{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "# Few parameters are not fully implemented yet\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "# not implemented but in the code in someparts\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "# load_model_ada = 0 \n",
    "# load_model_dnn = 0 \n",
    "# load_model_mlp = 0 \n",
    "# load_model_lgbm = 0 \n",
    "# load_model_rf = 0 \n",
    "# load_model_svm = 0\n",
    "# load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "# not implemented but in the code in someparts\n",
    "load_model_ada = 1\n",
    "load_model_dnn = 1 \n",
    "load_model_mlp = 1 \n",
    "load_model_lgbm = 1 \n",
    "load_model_rf = 1                               \n",
    "load_model_svm = 1\n",
    "load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "\n",
    "# Implemented\n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0 # OFF\n",
    "feature_selection_bit = 1 # On\n",
    "pick_prob = 1 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "# pick_prob = 0\n",
    "generate_feature_importance = 0 # Generate Shap graphs\n",
    "\n",
    "\n",
    "# column_features = [\n",
    "#                     # 'dnn',\n",
    "#                 #    'rf',\n",
    "#                    'lgbm',\n",
    "#                 #    'ada',\n",
    "#                    'knn',\n",
    "#                    'mlp',\n",
    "#                    'svm',\n",
    "#                 #    'cat',\n",
    "#                 #    'xgb',\n",
    "#                    'lr',\n",
    "#                    'dt',\n",
    "#                    'label']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_all_features_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_classes--', file = f)\n",
    "\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_all_features_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_probabilites--', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_classes--', file = f)\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_probabilites--', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "import time\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "\n",
    "import time\n",
    "start_program = time.time()\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    # voting_acc_01 = Acc\n",
    "    # voting_pre_01 = Precision\n",
    "    # weighed_avg_rec_01 = Recall\n",
    "    # weighed_avg_f1_01 = F1\n",
    "    # weighed_avg_bacc_01 = BACC\n",
    "    # weighed_avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.993925</td>\n",
       "      <td>0.953462</td>\n",
       "      <td>0.999990</td>\n",
       "      <td>0.241649</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.872094</td>\n",
       "      <td>0.996421</td>\n",
       "      <td>0.993838</td>\n",
       "      <td>0.999991</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.968751</td>\n",
       "      <td>0.755381</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.229428</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>0.972762</td>\n",
       "      <td>0.987883</td>\n",
       "      <td>0.974823</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.980010</td>\n",
       "      <td>0.931826</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.247061</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.125468</td>\n",
       "      <td>0.990564</td>\n",
       "      <td>0.979741</td>\n",
       "      <td>0.996046</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.698289</td>\n",
       "      <td>0.847230</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.242380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.028396</td>\n",
       "      <td>0.988092</td>\n",
       "      <td>0.993120</td>\n",
       "      <td>0.966385</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.991085</td>\n",
       "      <td>0.966415</td>\n",
       "      <td>0.999992</td>\n",
       "      <td>0.255779</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.916044</td>\n",
       "      <td>0.996865</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.999776</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44551</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>0.883479</td>\n",
       "      <td>0.999994</td>\n",
       "      <td>0.259007</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968749</td>\n",
       "      <td>0.994279</td>\n",
       "      <td>0.996916</td>\n",
       "      <td>0.999781</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>0.983328</td>\n",
       "      <td>0.983763</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.316873</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.009048</td>\n",
       "      <td>0.994402</td>\n",
       "      <td>0.997916</td>\n",
       "      <td>0.999954</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44553</th>\n",
       "      <td>0.984077</td>\n",
       "      <td>0.767309</td>\n",
       "      <td>0.999807</td>\n",
       "      <td>0.254683</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.944438</td>\n",
       "      <td>0.983802</td>\n",
       "      <td>0.976063</td>\n",
       "      <td>0.956723</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44554</th>\n",
       "      <td>0.343180</td>\n",
       "      <td>0.559281</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.292847</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000832</td>\n",
       "      <td>0.962144</td>\n",
       "      <td>0.976610</td>\n",
       "      <td>0.999991</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44555</th>\n",
       "      <td>0.999987</td>\n",
       "      <td>0.657648</td>\n",
       "      <td>0.999899</td>\n",
       "      <td>0.237328</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997933</td>\n",
       "      <td>0.977548</td>\n",
       "      <td>0.988660</td>\n",
       "      <td>0.999815</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44556 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
       "0      0.993925  0.953462  0.999990  0.241649  1.0  1.000000  0.872094   \n",
       "1      0.968751  0.755381  1.000000  0.229428  1.0  1.000000  0.000205   \n",
       "2      0.980010  0.931826  0.999936  0.247061  1.0  0.999999  0.125468   \n",
       "3      0.698289  0.847230  1.000000  0.242380  1.0  1.000000  0.028396   \n",
       "4      0.991085  0.966415  0.999992  0.255779  1.0  1.000000  0.916044   \n",
       "...         ...       ...       ...       ...  ...       ...       ...   \n",
       "44551  0.999963  0.883479  0.999994  0.259007  1.0  1.000000  0.968749   \n",
       "44552  0.983328  0.983763  1.000000  0.316873  1.0  1.000000  0.009048   \n",
       "44553  0.984077  0.767309  0.999807  0.254683  1.0  1.000000  0.944438   \n",
       "44554  0.343180  0.559281  0.999908  0.292847  1.0  1.000000  0.000832   \n",
       "44555  0.999987  0.657648  0.999899  0.237328  1.0  1.000000  0.997933   \n",
       "\n",
       "            cat       xgb        lr   dt  label  \n",
       "0      0.996421  0.993838  0.999991  1.0    0.0  \n",
       "1      0.972762  0.987883  0.974823  1.0    1.0  \n",
       "2      0.990564  0.979741  0.996046  1.0    0.0  \n",
       "3      0.988092  0.993120  0.966385  1.0    1.0  \n",
       "4      0.996865  0.994128  0.999776  1.0    0.0  \n",
       "...         ...       ...       ...  ...    ...  \n",
       "44551  0.994279  0.996916  0.999781  1.0    0.0  \n",
       "44552  0.994402  0.997916  0.999954  1.0    1.0  \n",
       "44553  0.983802  0.976063  0.956723  1.0    0.0  \n",
       "44554  0.962144  0.976610  0.999991  1.0    2.0  \n",
       "44555  0.977548  0.988660  0.999815  1.0    0.0  \n",
       "\n",
       "[44556 rows x 12 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = df_level_00_1.pop('label')\n",
    "X1 = df_level_00_1\n",
    "df_level_00_1 = X1.assign(label = y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "y0 = df_level_00_0.pop('label')\n",
    "X0 = df_level_00_0\n",
    "df_level_00_0 = X0.assign(label = y0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dt      1.002697\n",
      "mlp     0.991250\n",
      "xgb     0.987737\n",
      "cat     0.982763\n",
      "knn     0.972034\n",
      "lgbm    0.937509\n",
      "lr      0.910494\n",
      "svm     0.871758\n",
      "rf      0.774631\n",
      "dnn     0.535752\n",
      "ada     0.531235\n",
      "dtype: float64\n",
      "rf      0.968487\n",
      "ada     0.962392\n",
      "xgb     0.920791\n",
      "cat     0.718843\n",
      "dnn     0.583501\n",
      "lgbm    0.464928\n",
      "svm     0.441384\n",
      "lr      0.186982\n",
      "mlp     0.163151\n",
      "knn     0.021334\n",
      "dt      0.010563\n",
      "dtype: float64\n",
      "Top 5 feature names:\n",
      "['dt', 'mlp', 'xgb', 'cat', 'knn']\n",
      "['rf', 'ada', 'xgb', 'cat', 'dnn']\n"
     ]
    }
   ],
   "source": [
    "if feature_selection_bit == 1:\n",
    "\n",
    "    from sklearn.feature_selection import mutual_info_classif\n",
    "    %matplotlib inline\n",
    "\n",
    "    # Compute information gain using mutual information\n",
    "    importances0 = mutual_info_classif(X0, y0)\n",
    "    importances1 = mutual_info_classif(X1, y1)\n",
    "\n",
    "\n",
    "    feat_importances0 = pd.Series(importances0, df_level_00_0.columns[0:len(df_level_00_0.columns)-1])\n",
    "    feat_importances1= pd.Series(importances1, df_level_00_1.columns[0:len(df_level_00_1.columns)-1])\n",
    "\n",
    "    # feat_importances.plot(kind='barh', color = 'teal')\n",
    "        \n",
    "    feat_importances_sorted0 = feat_importances0.sort_values( ascending=False)\n",
    "    feat_importances_sorted1 = feat_importances1.sort_values( ascending=False)\n",
    "\n",
    "\n",
    "    # Print or use the sorted DataFrame\n",
    "    print(feat_importances_sorted0)\n",
    "    print(feat_importances_sorted1)\n",
    "\n",
    "    # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "    # feat_importances_sorted\n",
    "    top_features0 = feat_importances_sorted0.nlargest(5)\n",
    "    top_features1 = feat_importances_sorted1.nlargest(5)\n",
    "\n",
    "    top_feature_names0 = top_features0.index.tolist()\n",
    "    top_feature_names1 = top_features1.index.tolist()\n",
    "\n",
    "\n",
    "    print(\"Top 5 feature names:\")\n",
    "    print(top_feature_names0)\n",
    "    print(top_feature_names1)\n",
    "\n",
    "    column_features0 = top_feature_names0\n",
    "    column_features1 = top_feature_names1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "# if feature_selection_bit == 1:\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv',names=column_features)\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv',names=column_features)\n",
    "\n",
    "# if feature_selection_bit == 0:\n",
    "\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "# df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "# df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "if feature_selection_bit == 1:\n",
    "    df_level_00_0 = df_level_00_0[column_features0]\n",
    "    df_level_00_1 = df_level_00_1[column_features1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rf</th>\n",
       "      <th>ada</th>\n",
       "      <th>xgb</th>\n",
       "      <th>cat</th>\n",
       "      <th>dnn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.953462</td>\n",
       "      <td>0.241649</td>\n",
       "      <td>0.993838</td>\n",
       "      <td>0.996421</td>\n",
       "      <td>0.993925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.755381</td>\n",
       "      <td>0.229428</td>\n",
       "      <td>0.987883</td>\n",
       "      <td>0.972762</td>\n",
       "      <td>0.968751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.931826</td>\n",
       "      <td>0.247061</td>\n",
       "      <td>0.979741</td>\n",
       "      <td>0.990564</td>\n",
       "      <td>0.980010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.847230</td>\n",
       "      <td>0.242380</td>\n",
       "      <td>0.993120</td>\n",
       "      <td>0.988092</td>\n",
       "      <td>0.698289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.966415</td>\n",
       "      <td>0.255779</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.996865</td>\n",
       "      <td>0.991085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44551</th>\n",
       "      <td>0.883479</td>\n",
       "      <td>0.259007</td>\n",
       "      <td>0.996916</td>\n",
       "      <td>0.994279</td>\n",
       "      <td>0.999963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>0.983763</td>\n",
       "      <td>0.316873</td>\n",
       "      <td>0.997916</td>\n",
       "      <td>0.994402</td>\n",
       "      <td>0.983328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44553</th>\n",
       "      <td>0.767309</td>\n",
       "      <td>0.254683</td>\n",
       "      <td>0.976063</td>\n",
       "      <td>0.983802</td>\n",
       "      <td>0.984077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44554</th>\n",
       "      <td>0.559281</td>\n",
       "      <td>0.292847</td>\n",
       "      <td>0.976610</td>\n",
       "      <td>0.962144</td>\n",
       "      <td>0.343180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44555</th>\n",
       "      <td>0.657648</td>\n",
       "      <td>0.237328</td>\n",
       "      <td>0.988660</td>\n",
       "      <td>0.977548</td>\n",
       "      <td>0.999987</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44556 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             rf       ada       xgb       cat       dnn\n",
       "0      0.953462  0.241649  0.993838  0.996421  0.993925\n",
       "1      0.755381  0.229428  0.987883  0.972762  0.968751\n",
       "2      0.931826  0.247061  0.979741  0.990564  0.980010\n",
       "3      0.847230  0.242380  0.993120  0.988092  0.698289\n",
       "4      0.966415  0.255779  0.994128  0.996865  0.991085\n",
       "...         ...       ...       ...       ...       ...\n",
       "44551  0.883479  0.259007  0.996916  0.994279  0.999963\n",
       "44552  0.983763  0.316873  0.997916  0.994402  0.983328\n",
       "44553  0.767309  0.254683  0.976063  0.983802  0.984077\n",
       "44554  0.559281  0.292847  0.976610  0.962144  0.343180\n",
       "44555  0.657648  0.237328  0.988660  0.977548  0.999987\n",
       "\n",
       "[44556 rows x 5 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dt</th>\n",
       "      <th>mlp</th>\n",
       "      <th>xgb</th>\n",
       "      <th>cat</th>\n",
       "      <th>knn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44551</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44553</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44554</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44555</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44556 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        dt  mlp  xgb  cat  knn\n",
       "0      0.0  0.0  0.0  0.0  0.0\n",
       "1      1.0  1.0  1.0  1.0  1.0\n",
       "2      0.0  0.0  0.0  0.0  0.0\n",
       "3      1.0  1.0  1.0  1.0  1.0\n",
       "4      0.0  0.0  0.0  0.0  0.0\n",
       "...    ...  ...  ...  ...  ...\n",
       "44551  0.0  0.0  0.0  0.0  0.0\n",
       "44552  1.0  1.0  1.0  1.0  1.0\n",
       "44553  0.0  0.0  0.0  0.0  0.0\n",
       "44554  2.0  2.0  2.0  2.0  2.0\n",
       "44555  0.0  0.0  0.0  0.0  0.0\n",
       "\n",
       "[44556 rows x 5 columns]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dt</th>\n",
       "      <th>mlp</th>\n",
       "      <th>xgb</th>\n",
       "      <th>cat</th>\n",
       "      <th>knn</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44551</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44553</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44554</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44555</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44556 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        dt  mlp  xgb  cat  knn  label\n",
       "0      0.0  0.0  0.0  0.0  0.0    0.0\n",
       "1      1.0  1.0  1.0  1.0  1.0    1.0\n",
       "2      0.0  0.0  0.0  0.0  0.0    0.0\n",
       "3      1.0  1.0  1.0  1.0  1.0    1.0\n",
       "4      1.0  1.0  1.0  1.0  1.0    1.0\n",
       "...    ...  ...  ...  ...  ...    ...\n",
       "44551  3.0  3.0  0.0  3.0  3.0    3.0\n",
       "44552  2.0  2.0  2.0  2.0  2.0    2.0\n",
       "44553  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "44554  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "44555  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "\n",
       "[44556 rows x 6 columns]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pick_prob == 1:\n",
    "    df_level_01 = df_level_00_1\n",
    "else: \n",
    "    df_level_01 = df_level_00_0\n",
    "\n",
    "df_level_01 = df_level_01.assign(label = y1)\n",
    "\n",
    "y_01 = df_level_01.pop('label') \n",
    "    \n",
    "X_01 = df_level_01\n",
    "df_level_01 = df_level_01.assign(label = y_01)\n",
    "\n",
    "\n",
    "split = 0.7\n",
    "X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# df_level_02 = pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "# df_level_02\n",
    "\n",
    "# y_02 = df_level_02.pop('label')\n",
    "# X_02 = df_level_02\n",
    "# df_level_02 = df_level_02.assign(label = y_01)\n",
    "\n",
    "\n",
    "# split = 0.7\n",
    "# X_train_02,X_test_02, y_train_02, y_test_02 = sklearn.model_selection.train_test_split(X_02, y_02, train_size=split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the stronger model - STACK level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#----------------------------------------------------------------\n",
    "with open(output_file_name, \"a\") as f: print('Stack model - Strong learner - level 01', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rf</th>\n",
       "      <th>ada</th>\n",
       "      <th>xgb</th>\n",
       "      <th>cat</th>\n",
       "      <th>dnn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4156</th>\n",
       "      <td>0.994399</td>\n",
       "      <td>0.307118</td>\n",
       "      <td>0.998120</td>\n",
       "      <td>0.997257</td>\n",
       "      <td>0.998582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15237</th>\n",
       "      <td>0.953462</td>\n",
       "      <td>0.255779</td>\n",
       "      <td>0.994217</td>\n",
       "      <td>0.996436</td>\n",
       "      <td>0.993875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2904</th>\n",
       "      <td>0.961571</td>\n",
       "      <td>0.257206</td>\n",
       "      <td>0.992883</td>\n",
       "      <td>0.996283</td>\n",
       "      <td>0.955951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35696</th>\n",
       "      <td>0.696609</td>\n",
       "      <td>0.318714</td>\n",
       "      <td>0.992532</td>\n",
       "      <td>0.996331</td>\n",
       "      <td>0.331834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39603</th>\n",
       "      <td>0.966415</td>\n",
       "      <td>0.255779</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.996854</td>\n",
       "      <td>0.979609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11567</th>\n",
       "      <td>0.898977</td>\n",
       "      <td>0.259007</td>\n",
       "      <td>0.996916</td>\n",
       "      <td>0.993961</td>\n",
       "      <td>0.999960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42730</th>\n",
       "      <td>0.961571</td>\n",
       "      <td>0.257206</td>\n",
       "      <td>0.992883</td>\n",
       "      <td>0.996643</td>\n",
       "      <td>0.975372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35749</th>\n",
       "      <td>0.724900</td>\n",
       "      <td>0.305816</td>\n",
       "      <td>0.886316</td>\n",
       "      <td>0.874074</td>\n",
       "      <td>0.288488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32359</th>\n",
       "      <td>0.919764</td>\n",
       "      <td>0.259007</td>\n",
       "      <td>0.994849</td>\n",
       "      <td>0.994746</td>\n",
       "      <td>0.999964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39135</th>\n",
       "      <td>0.449727</td>\n",
       "      <td>0.310847</td>\n",
       "      <td>0.983313</td>\n",
       "      <td>0.925396</td>\n",
       "      <td>0.330611</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13367 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             rf       ada       xgb       cat       dnn\n",
       "4156   0.994399  0.307118  0.998120  0.997257  0.998582\n",
       "15237  0.953462  0.255779  0.994217  0.996436  0.993875\n",
       "2904   0.961571  0.257206  0.992883  0.996283  0.955951\n",
       "35696  0.696609  0.318714  0.992532  0.996331  0.331834\n",
       "39603  0.966415  0.255779  0.994128  0.996854  0.979609\n",
       "...         ...       ...       ...       ...       ...\n",
       "11567  0.898977  0.259007  0.996916  0.993961  0.999960\n",
       "42730  0.961571  0.257206  0.992883  0.996643  0.975372\n",
       "35749  0.724900  0.305816  0.886316  0.874074  0.288488\n",
       "32359  0.919764  0.259007  0.994849  0.994746  0.999964\n",
       "39135  0.449727  0.310847  0.983313  0.925396  0.330611\n",
       "\n",
       "[13367 rows x 5 columns]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "    \n",
    "if pick_prob == 0:\n",
    "    # Voting start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    "    print(df)\n",
    "\n",
    "    pred_label = df ['ensemble'].values\n",
    "    df.pop('ensemble')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'voting'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "   \n",
    "else:\n",
    "    name = 'voting'\n",
    "    globals()[f\"{name}_acc_01\"] = 0\n",
    "    globals()[f\"{name}_pre_01\"] = 0\n",
    "    globals()[f\"{name}_rec_01\"] = 0\n",
    "    globals()[f\"{name}_f1_01\"] = 0\n",
    "    globals()[f\"{name}_bacc_01\"] = 0\n",
    "    globals()[f\"{name}_mcc_01\"] = 0\n",
    "    globals()[f\"{name}_time_01\"] = 9999\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_acc_01\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             rf       ada       xgb       cat       dnn  results\n",
      "4156   0.994399  0.307118  0.998120  0.997257  0.998582        1\n",
      "15237  0.953462  0.255779  0.994217  0.996436  0.993875        1\n",
      "2904   0.961571  0.257206  0.992883  0.996283  0.955951        1\n",
      "35696  0.696609  0.318714  0.992532  0.996331  0.331834        1\n",
      "39603  0.966415  0.255779  0.994128  0.996854  0.979609        1\n",
      "...         ...       ...       ...       ...       ...      ...\n",
      "11567  0.898977  0.259007  0.996916  0.993961  0.999960        1\n",
      "42730  0.961571  0.257206  0.992883  0.996643  0.975372        1\n",
      "35749  0.724900  0.305816  0.886316  0.874074  0.288488        1\n",
      "32359  0.919764  0.259007  0.994849  0.994746  0.999964        1\n",
      "39135  0.449727  0.310847  0.983313  0.925396  0.330611        1\n",
      "\n",
      "[13367 rows x 6 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0     1.0  2.0  3.0  4.0\n",
      "0.0   56.0  6824.0  0.0  0.0  0.0\n",
      "1.0  101.0  4734.0  0.0  0.0  0.0\n",
      "2.0  130.0  1163.0  0.0  0.0  0.0\n",
      "3.0  236.0   113.0  0.0  0.0  0.0\n",
      "4.0   10.0     0.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.3583451784244782\n",
      "Precision total:  0.09478592423209808\n",
      "Recall total:  0.19745003727664076\n",
      "F1 total:  0.11019246963533762\n",
      "BACC total:  0.19745003727664076\n",
      "MCC total:  -0.0443821959109148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# if pick_prob == 0:\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "   \n",
    "\n",
    "    column_sums = df.sum(axis=1)\n",
    "    row_average = df.mean(axis=1)\n",
    "\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_average = row_average.round().astype(int)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    name = 'avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    \n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighed Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column_features\n",
    "# move this up with column_features\n",
    "\n",
    "#important update this as you need to select the important features,\n",
    "#  the left is the least important while the right is the most important\n",
    "# needs automation\n",
    "if pick_prob == 1:\n",
    "    column_features = column_features1\n",
    "else: column_features = column_features0\n",
    "feature_selection_columns_in_order_of_importance = column_features[:-1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.3333333333333333, 0.6666666666666666, 1.0]\n",
      "[0.0, 0.3333333333333333, 0.6666666666666666, 1.0]\n",
      "             rf       ada       xgb       cat\n",
      "4156   0.994399  0.307118  0.998120  0.997257\n",
      "15237  0.953462  0.255779  0.994217  0.996436\n",
      "2904   0.961571  0.257206  0.992883  0.996283\n",
      "35696  0.696609  0.318714  0.992532  0.996331\n",
      "39603  0.966415  0.255779  0.994128  0.996854\n",
      "...         ...       ...       ...       ...\n",
      "11567  0.898977  0.259007  0.996916  0.993961\n",
      "42730  0.961571  0.257206  0.992883  0.996643\n",
      "35749  0.724900  0.305816  0.886316  0.874074\n",
      "32359  0.919764  0.259007  0.994849  0.994746\n",
      "39135  0.449727  0.310847  0.983313  0.925396\n",
      "\n",
      "[13367 rows x 4 columns]\n",
      "4156     0.882521\n",
      "15237    0.872254\n",
      "2904     0.871970\n",
      "35696    0.882128\n",
      "39603    0.872433\n",
      "           ...   \n",
      "11567    0.872453\n",
      "42730    0.872150\n",
      "35749    0.783445\n",
      "32359    0.872157\n",
      "39135    0.842277\n",
      "Length: 13367, dtype: float64\n",
      "4156     1\n",
      "15237    1\n",
      "2904     1\n",
      "35696    1\n",
      "39603    1\n",
      "        ..\n",
      "11567    1\n",
      "42730    1\n",
      "35749    1\n",
      "32359    1\n",
      "39135    1\n",
      "Length: 13367, dtype: int64\n",
      "             rf       ada       xgb       cat  results\n",
      "4156   0.994399  0.307118  0.998120  0.997257        1\n",
      "15237  0.953462  0.255779  0.994217  0.996436        1\n",
      "2904   0.961571  0.257206  0.992883  0.996283        1\n",
      "35696  0.696609  0.318714  0.992532  0.996331        1\n",
      "39603  0.966415  0.255779  0.994128  0.996854        1\n",
      "...         ...       ...       ...       ...      ...\n",
      "11567  0.898977  0.259007  0.996916  0.993961        1\n",
      "42730  0.961571  0.257206  0.992883  0.996643        1\n",
      "35749  0.724900  0.305816  0.886316  0.874074        1\n",
      "32359  0.919764  0.259007  0.994849  0.994746        1\n",
      "39135  0.449727  0.310847  0.983313  0.925396        1\n",
      "\n",
      "[13367 rows x 5 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0     1.0  2.0  3.0  4.0\n",
      "0.0  25.0  6855.0  0.0  0.0  0.0\n",
      "1.0   9.0  4826.0  0.0  0.0  0.0\n",
      "2.0  23.0  1270.0  0.0  0.0  0.0\n",
      "3.0  22.0   327.0  0.0  0.0  0.0\n",
      "4.0   6.0     4.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.36290865564449765\n",
      "Precision total:  0.1314933080595587\n",
      "Recall total:  0.2003544587672254\n",
      "F1 total:  0.1079876075509983\n",
      "BACC total:  0.2003544587672254\n",
      "MCC total:  0.0025864863630347726\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# if pick_prob == 0:\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    # df = X_test_01\n",
    "    df = X_test_01[feature_selection_columns_in_order_of_importance]\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # weight\n",
    "    weights_values = []\n",
    "\n",
    "    # linear weight distribution\n",
    "    for i in range(0,len(~df.columns.isin(['label']))):\n",
    "        weights_values.append(i/(len(~df.columns.isin(['label']))-1))\n",
    "    print(weights_values)\n",
    "    # weights_values = [10,3,2,2.3]\n",
    "    print(weights_values)\n",
    "    print(df)\n",
    "    weighted_average = df.multiply(weights_values).sum(axis=1) / sum(weights_values)\n",
    "    print(weighted_average)\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_weighted_average = weighted_average.round().astype(int)\n",
    "\n",
    "    print(rounded_weighted_average)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_weighted_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'weighed_avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6841.0    17.0    20.0    2.0  0.0\n",
      "1.0    30.0  4790.0    12.0    3.0  0.0\n",
      "2.0     4.0     7.0  1276.0    5.0  1.0\n",
      "3.0     5.0     1.0    11.0  331.0  1.0\n",
      "4.0     0.0     0.0     1.0    1.0  8.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9909478566619286\n",
      "Precision total:  0.9447284422659198\n",
      "Recall total:  0.9440601220324145\n",
      "F1 total:  0.9443529065729896\n",
      "BACC total:  0.9440601220324145\n",
      "MCC total:  0.9847809113663472\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "start = time.time()\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_dt'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with SVM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6520.0   174.0   173.0   13.0  0.0\n",
      "1.0  1255.0  3360.0   157.0   63.0  0.0\n",
      "2.0    39.0    18.0  1140.0   93.0  3.0\n",
      "3.0    11.0     0.0   109.0  229.0  0.0\n",
      "4.0     4.0     0.0     1.0    5.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8415500860327673\n",
      "Precision total:  0.6137008541426557\n",
      "Recall total:  0.6360876384998889\n",
      "F1 total:  0.618082033486082\n",
      "BACC total:  0.6360876384998889\n",
      "MCC total:  0.7409188865171652\n"
     ]
    }
   ],
   "source": [
    "## bagging  with SVM\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.keras.models import Sequential\n",
    "# from tensorflow.keras.layers import Dense\n",
    "\n",
    "# #Model Parameters\n",
    "# dropout_rate = 0.2\n",
    "# nodes = 3\n",
    "# out_layer = 5\n",
    "# optimizer='adam'\n",
    "# loss='sparse_categorical_crossentropy'\n",
    "# epochs=100\n",
    "# batch_size=128\n",
    "\n",
    "\n",
    "# num_columns = X_train_01.shape[1]\n",
    "\n",
    "# dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# # Input layer\n",
    "# dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# # Output layer\n",
    "# # dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "# dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "# base_classifier = dnn_01\n",
    "\n",
    "# # Define the BaggingClassifier\n",
    "# bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# # Train the BaggingClassifier\n",
    "# bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# # Evaluate accuracy\n",
    "# # accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# # print(f'Accuracy: {accuracy}')\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Bagging with DNN', file = f)\n",
    "\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('CONFUSION MATRIX')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# pred_label = y_pred\n",
    "\n",
    "# confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "# all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "# z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "# rows, cols = confusion_matrix.shape\n",
    "# z[:rows, :cols] = confusion_matrix\n",
    "# confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "# print(confusion_matrix)\n",
    "# with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "# FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "# FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "# TP = np.diag(confusion_matrix)\n",
    "# TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "# TP_total = sum(TP)\n",
    "# TN_total = sum(TN)\n",
    "# FP_total = sum(FP)\n",
    "# FN_total = sum(FN)\n",
    "\n",
    "# TP_total = np.array(TP_total,dtype=np.float64)\n",
    "# TN_total = np.array(TN_total,dtype=np.float64)\n",
    "# FP_total = np.array(FP_total,dtype=np.float64)\n",
    "# FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "# #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('METRICS')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# Acc = accuracy_score(y_test_01, pred_label)\n",
    "# Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "# Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "# F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "# BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "# MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "# bag_dnn_acc_01 = Acc\n",
    "# bag_dnn_pre_01 = Precision\n",
    "# bag_dnn_rec_01 = Recall\n",
    "# bag_dnn_f1_01 = F1\n",
    "# bag_dnn_bacc_01 = BACC\n",
    "# bag_dnn_mcc_01 = MCC\n",
    "# # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "# print('Accuracy total: ', Acc)\n",
    "# print('Precision total: ', Precision )\n",
    "# print('Recall total: ', Recall )\n",
    "# print('F1 total: ', F1 )\n",
    "# print('BACC total: ', BACC)\n",
    "# print('MCC total: ', MCC)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6544.0   264.0    66.0    6.0  0.0\n",
      "1.0   419.0  4369.0    42.0    5.0  0.0\n",
      "2.0     9.0    11.0  1195.0   78.0  0.0\n",
      "3.0     3.0     2.0     9.0  332.0  3.0\n",
      "4.0     0.0     3.0     0.0    2.0  5.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9310241639859355\n",
      "Precision total:  0.8397346052745809\n",
      "Recall total:  0.846055780093055\n",
      "F1 total:  0.8398215446686835\n",
      "BACC total:  0.846055780093055\n",
      "MCC total:  0.8843398541664509\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "start = time.time()\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp_01 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6787.0    55.0    36.0    2.0  0.0\n",
      "1.0    43.0  4779.0    11.0    2.0  0.0\n",
      "2.0     6.0    11.0  1260.0   15.0  1.0\n",
      "3.0     5.0     0.0     5.0  338.0  1.0\n",
      "4.0     0.0     0.0     0.0    4.0  6.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9852622129123962\n",
      "Precision total:  0.9250276136939988\n",
      "Recall total:  0.9035719357408741\n",
      "F1 total:  0.9125665947100057\n",
      "BACC total:  0.9035719357408741\n",
      "MCC total:  0.9752535589024215\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "\n",
    "base_classifier = knn_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging LogRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6522.0   217.0   128.0   13.0  0.0\n",
      "1.0   890.0  3764.0   102.0   79.0  0.0\n",
      "2.0    27.0    15.0  1192.0   59.0  0.0\n",
      "3.0    10.0     0.0    99.0  240.0  0.0\n",
      "4.0     3.0     0.0     2.0    5.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8766364928555398\n",
      "Precision total:  0.6411739254897781\n",
      "Recall total:  0.6672042918950302\n",
      "F1 total:  0.6506937954167306\n",
      "BACC total:  0.6672042918950302\n",
      "MCC total:  0.7961128045169892\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "start = time.time()\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0     3.0  4.0\n",
      "0.0  17.0   0.0  20.0  6843.0  0.0\n",
      "1.0   5.0  39.0  65.0  4726.0  0.0\n",
      "2.0  19.0   3.0  96.0  1174.0  1.0\n",
      "3.0  44.0   2.0  88.0   214.0  1.0\n",
      "4.0   0.0   0.0   1.0     0.0  9.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.028054163237824493\n",
      "Precision total:  0.455323435793452\n",
      "Recall total:  0.3195927139483003\n",
      "F1 total:  0.20660358429344053\n",
      "BACC total:  0.3195927139483003\n",
      "MCC total:  -0.019457045713664144\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging CAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3055063\ttotal: 14.2ms\tremaining: 1.41s\n",
      "1:\tlearn: 1.1071924\ttotal: 26.2ms\tremaining: 1.28s\n",
      "2:\tlearn: 0.9642307\ttotal: 36.4ms\tremaining: 1.18s\n",
      "3:\tlearn: 0.8447270\ttotal: 45.8ms\tremaining: 1.1s\n",
      "4:\tlearn: 0.7520087\ttotal: 54.9ms\tremaining: 1.04s\n",
      "5:\tlearn: 0.6729947\ttotal: 62.5ms\tremaining: 980ms\n",
      "6:\tlearn: 0.6072709\ttotal: 70ms\tremaining: 930ms\n",
      "7:\tlearn: 0.5495889\ttotal: 77.1ms\tremaining: 887ms\n",
      "8:\tlearn: 0.5025959\ttotal: 83.6ms\tremaining: 845ms\n",
      "9:\tlearn: 0.4589638\ttotal: 90.2ms\tremaining: 812ms\n",
      "10:\tlearn: 0.4204864\ttotal: 95.9ms\tremaining: 776ms\n",
      "11:\tlearn: 0.3889704\ttotal: 102ms\tremaining: 745ms\n",
      "12:\tlearn: 0.3586915\ttotal: 107ms\tremaining: 719ms\n",
      "13:\tlearn: 0.3340234\ttotal: 113ms\tremaining: 693ms\n",
      "14:\tlearn: 0.3124581\ttotal: 119ms\tremaining: 672ms\n",
      "15:\tlearn: 0.2927232\ttotal: 124ms\tremaining: 649ms\n",
      "16:\tlearn: 0.2738716\ttotal: 129ms\tremaining: 628ms\n",
      "17:\tlearn: 0.2574387\ttotal: 134ms\tremaining: 609ms\n",
      "18:\tlearn: 0.2430068\ttotal: 139ms\tremaining: 591ms\n",
      "19:\tlearn: 0.2289335\ttotal: 144ms\tremaining: 577ms\n",
      "20:\tlearn: 0.2158838\ttotal: 149ms\tremaining: 562ms\n",
      "21:\tlearn: 0.2047761\ttotal: 154ms\tremaining: 547ms\n",
      "22:\tlearn: 0.1943364\ttotal: 159ms\tremaining: 533ms\n",
      "23:\tlearn: 0.1847090\ttotal: 164ms\tremaining: 520ms\n",
      "24:\tlearn: 0.1758108\ttotal: 169ms\tremaining: 508ms\n",
      "25:\tlearn: 0.1683457\ttotal: 174ms\tremaining: 496ms\n",
      "26:\tlearn: 0.1615759\ttotal: 179ms\tremaining: 483ms\n",
      "27:\tlearn: 0.1553983\ttotal: 183ms\tremaining: 471ms\n",
      "28:\tlearn: 0.1489672\ttotal: 188ms\tremaining: 460ms\n",
      "29:\tlearn: 0.1435958\ttotal: 193ms\tremaining: 449ms\n",
      "30:\tlearn: 0.1381280\ttotal: 197ms\tremaining: 438ms\n",
      "31:\tlearn: 0.1336686\ttotal: 202ms\tremaining: 429ms\n",
      "32:\tlearn: 0.1291521\ttotal: 207ms\tremaining: 419ms\n",
      "33:\tlearn: 0.1253697\ttotal: 212ms\tremaining: 411ms\n",
      "34:\tlearn: 0.1219414\ttotal: 216ms\tremaining: 402ms\n",
      "35:\tlearn: 0.1184622\ttotal: 221ms\tremaining: 394ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36:\tlearn: 0.1157594\ttotal: 227ms\tremaining: 386ms\n",
      "37:\tlearn: 0.1132928\ttotal: 231ms\tremaining: 377ms\n",
      "38:\tlearn: 0.1107769\ttotal: 236ms\tremaining: 369ms\n",
      "39:\tlearn: 0.1086397\ttotal: 240ms\tremaining: 361ms\n",
      "40:\tlearn: 0.1050512\ttotal: 246ms\tremaining: 354ms\n",
      "41:\tlearn: 0.1030086\ttotal: 250ms\tremaining: 346ms\n",
      "42:\tlearn: 0.1007605\ttotal: 255ms\tremaining: 338ms\n",
      "43:\tlearn: 0.0990677\ttotal: 260ms\tremaining: 331ms\n",
      "44:\tlearn: 0.0967195\ttotal: 265ms\tremaining: 323ms\n",
      "45:\tlearn: 0.0946729\ttotal: 269ms\tremaining: 316ms\n",
      "46:\tlearn: 0.0924936\ttotal: 274ms\tremaining: 309ms\n",
      "47:\tlearn: 0.0913791\ttotal: 279ms\tremaining: 302ms\n",
      "48:\tlearn: 0.0903176\ttotal: 284ms\tremaining: 295ms\n",
      "49:\tlearn: 0.0889566\ttotal: 289ms\tremaining: 289ms\n",
      "50:\tlearn: 0.0874924\ttotal: 293ms\tremaining: 281ms\n",
      "51:\tlearn: 0.0857907\ttotal: 297ms\tremaining: 274ms\n",
      "52:\tlearn: 0.0846626\ttotal: 302ms\tremaining: 268ms\n",
      "53:\tlearn: 0.0825404\ttotal: 306ms\tremaining: 261ms\n",
      "54:\tlearn: 0.0812352\ttotal: 311ms\tremaining: 254ms\n",
      "55:\tlearn: 0.0807169\ttotal: 315ms\tremaining: 248ms\n",
      "56:\tlearn: 0.0800945\ttotal: 320ms\tremaining: 241ms\n",
      "57:\tlearn: 0.0791376\ttotal: 324ms\tremaining: 235ms\n",
      "58:\tlearn: 0.0778083\ttotal: 329ms\tremaining: 229ms\n",
      "59:\tlearn: 0.0756222\ttotal: 334ms\tremaining: 223ms\n",
      "60:\tlearn: 0.0740661\ttotal: 339ms\tremaining: 217ms\n",
      "61:\tlearn: 0.0724433\ttotal: 344ms\tremaining: 211ms\n",
      "62:\tlearn: 0.0712317\ttotal: 349ms\tremaining: 205ms\n",
      "63:\tlearn: 0.0706216\ttotal: 354ms\tremaining: 199ms\n",
      "64:\tlearn: 0.0700818\ttotal: 358ms\tremaining: 193ms\n",
      "65:\tlearn: 0.0691951\ttotal: 363ms\tremaining: 187ms\n",
      "66:\tlearn: 0.0686656\ttotal: 367ms\tremaining: 181ms\n",
      "67:\tlearn: 0.0681091\ttotal: 372ms\tremaining: 175ms\n",
      "68:\tlearn: 0.0673794\ttotal: 377ms\tremaining: 169ms\n",
      "69:\tlearn: 0.0667146\ttotal: 381ms\tremaining: 163ms\n",
      "70:\tlearn: 0.0654540\ttotal: 386ms\tremaining: 157ms\n",
      "71:\tlearn: 0.0648652\ttotal: 390ms\tremaining: 152ms\n",
      "72:\tlearn: 0.0639007\ttotal: 396ms\tremaining: 146ms\n",
      "73:\tlearn: 0.0633541\ttotal: 400ms\tremaining: 141ms\n",
      "74:\tlearn: 0.0626266\ttotal: 405ms\tremaining: 135ms\n",
      "75:\tlearn: 0.0620835\ttotal: 409ms\tremaining: 129ms\n",
      "76:\tlearn: 0.0612097\ttotal: 423ms\tremaining: 126ms\n",
      "77:\tlearn: 0.0601920\ttotal: 428ms\tremaining: 121ms\n",
      "78:\tlearn: 0.0597552\ttotal: 432ms\tremaining: 115ms\n",
      "79:\tlearn: 0.0593900\ttotal: 437ms\tremaining: 109ms\n",
      "80:\tlearn: 0.0587790\ttotal: 441ms\tremaining: 103ms\n",
      "81:\tlearn: 0.0578959\ttotal: 446ms\tremaining: 97.9ms\n",
      "82:\tlearn: 0.0575564\ttotal: 450ms\tremaining: 92.2ms\n",
      "83:\tlearn: 0.0566047\ttotal: 455ms\tremaining: 86.7ms\n",
      "84:\tlearn: 0.0563039\ttotal: 460ms\tremaining: 81.1ms\n",
      "85:\tlearn: 0.0554923\ttotal: 464ms\tremaining: 75.6ms\n",
      "86:\tlearn: 0.0549128\ttotal: 469ms\tremaining: 70.1ms\n",
      "87:\tlearn: 0.0546772\ttotal: 473ms\tremaining: 64.5ms\n",
      "88:\tlearn: 0.0542208\ttotal: 478ms\tremaining: 59ms\n",
      "89:\tlearn: 0.0538743\ttotal: 482ms\tremaining: 53.5ms\n",
      "90:\tlearn: 0.0534792\ttotal: 487ms\tremaining: 48.1ms\n",
      "91:\tlearn: 0.0530294\ttotal: 492ms\tremaining: 42.7ms\n",
      "92:\tlearn: 0.0526162\ttotal: 496ms\tremaining: 37.3ms\n",
      "93:\tlearn: 0.0521972\ttotal: 501ms\tremaining: 32ms\n",
      "94:\tlearn: 0.0516702\ttotal: 506ms\tremaining: 26.6ms\n",
      "95:\tlearn: 0.0512919\ttotal: 510ms\tremaining: 21.3ms\n",
      "96:\tlearn: 0.0510221\ttotal: 515ms\tremaining: 15.9ms\n",
      "97:\tlearn: 0.0508582\ttotal: 519ms\tremaining: 10.6ms\n",
      "98:\tlearn: 0.0505321\ttotal: 524ms\tremaining: 5.29ms\n",
      "99:\tlearn: 0.0499299\ttotal: 528ms\tremaining: 0us\n",
      "0:\tlearn: 1.3140774\ttotal: 12.8ms\tremaining: 1.26s\n",
      "1:\tlearn: 1.1174996\ttotal: 23ms\tremaining: 1.13s\n",
      "2:\tlearn: 0.9748341\ttotal: 33.3ms\tremaining: 1.08s\n",
      "3:\tlearn: 0.8564376\ttotal: 43.5ms\tremaining: 1.04s\n",
      "4:\tlearn: 0.7617764\ttotal: 52ms\tremaining: 988ms\n",
      "5:\tlearn: 0.6806790\ttotal: 61.8ms\tremaining: 969ms\n",
      "6:\tlearn: 0.6151894\ttotal: 70.7ms\tremaining: 940ms\n",
      "7:\tlearn: 0.5556325\ttotal: 79.3ms\tremaining: 912ms\n",
      "8:\tlearn: 0.5075814\ttotal: 87.1ms\tremaining: 881ms\n",
      "9:\tlearn: 0.4654318\ttotal: 94.8ms\tremaining: 853ms\n",
      "10:\tlearn: 0.4259443\ttotal: 102ms\tremaining: 825ms\n",
      "11:\tlearn: 0.3920554\ttotal: 109ms\tremaining: 799ms\n",
      "12:\tlearn: 0.3619648\ttotal: 116ms\tremaining: 775ms\n",
      "13:\tlearn: 0.3360123\ttotal: 123ms\tremaining: 754ms\n",
      "14:\tlearn: 0.3142179\ttotal: 129ms\tremaining: 732ms\n",
      "15:\tlearn: 0.2938524\ttotal: 136ms\tremaining: 712ms\n",
      "16:\tlearn: 0.2758005\ttotal: 142ms\tremaining: 691ms\n",
      "17:\tlearn: 0.2594576\ttotal: 148ms\tremaining: 674ms\n",
      "18:\tlearn: 0.2452317\ttotal: 154ms\tremaining: 655ms\n",
      "19:\tlearn: 0.2309381\ttotal: 159ms\tremaining: 638ms\n",
      "20:\tlearn: 0.2182607\ttotal: 166ms\tremaining: 624ms\n",
      "21:\tlearn: 0.2065000\ttotal: 171ms\tremaining: 608ms\n",
      "22:\tlearn: 0.1962880\ttotal: 177ms\tremaining: 591ms\n",
      "23:\tlearn: 0.1866314\ttotal: 183ms\tremaining: 578ms\n",
      "24:\tlearn: 0.1777391\ttotal: 188ms\tremaining: 564ms\n",
      "25:\tlearn: 0.1694643\ttotal: 194ms\tremaining: 551ms\n",
      "26:\tlearn: 0.1629958\ttotal: 199ms\tremaining: 538ms\n",
      "27:\tlearn: 0.1561424\ttotal: 204ms\tremaining: 524ms\n",
      "28:\tlearn: 0.1499729\ttotal: 209ms\tremaining: 512ms\n",
      "29:\tlearn: 0.1451854\ttotal: 214ms\tremaining: 499ms\n",
      "30:\tlearn: 0.1396507\ttotal: 219ms\tremaining: 487ms\n",
      "31:\tlearn: 0.1346953\ttotal: 224ms\tremaining: 476ms\n",
      "32:\tlearn: 0.1294172\ttotal: 230ms\tremaining: 466ms\n",
      "33:\tlearn: 0.1252955\ttotal: 235ms\tremaining: 456ms\n",
      "34:\tlearn: 0.1219600\ttotal: 239ms\tremaining: 444ms\n",
      "35:\tlearn: 0.1189998\ttotal: 244ms\tremaining: 433ms\n",
      "36:\tlearn: 0.1156227\ttotal: 248ms\tremaining: 423ms\n",
      "37:\tlearn: 0.1124097\ttotal: 254ms\tremaining: 414ms\n",
      "38:\tlearn: 0.1091533\ttotal: 258ms\tremaining: 404ms\n",
      "39:\tlearn: 0.1074355\ttotal: 262ms\tremaining: 394ms\n",
      "40:\tlearn: 0.1041867\ttotal: 267ms\tremaining: 385ms\n",
      "41:\tlearn: 0.1017677\ttotal: 272ms\tremaining: 375ms\n",
      "42:\tlearn: 0.0997861\ttotal: 277ms\tremaining: 367ms\n",
      "43:\tlearn: 0.0975706\ttotal: 281ms\tremaining: 358ms\n",
      "44:\tlearn: 0.0950693\ttotal: 286ms\tremaining: 349ms\n",
      "45:\tlearn: 0.0931263\ttotal: 290ms\tremaining: 341ms\n",
      "46:\tlearn: 0.0916795\ttotal: 295ms\tremaining: 332ms\n",
      "47:\tlearn: 0.0896133\ttotal: 300ms\tremaining: 324ms\n",
      "48:\tlearn: 0.0873581\ttotal: 305ms\tremaining: 317ms\n",
      "49:\tlearn: 0.0865128\ttotal: 309ms\tremaining: 309ms\n",
      "50:\tlearn: 0.0853314\ttotal: 313ms\tremaining: 301ms\n",
      "51:\tlearn: 0.0834989\ttotal: 318ms\tremaining: 294ms\n",
      "52:\tlearn: 0.0827416\ttotal: 322ms\tremaining: 286ms\n",
      "53:\tlearn: 0.0810226\ttotal: 327ms\tremaining: 279ms\n",
      "54:\tlearn: 0.0789975\ttotal: 332ms\tremaining: 272ms\n",
      "55:\tlearn: 0.0778747\ttotal: 337ms\tremaining: 265ms\n",
      "56:\tlearn: 0.0768362\ttotal: 341ms\tremaining: 257ms\n",
      "57:\tlearn: 0.0754612\ttotal: 346ms\tremaining: 250ms\n",
      "58:\tlearn: 0.0748838\ttotal: 350ms\tremaining: 243ms\n",
      "59:\tlearn: 0.0728375\ttotal: 355ms\tremaining: 237ms\n",
      "60:\tlearn: 0.0714025\ttotal: 360ms\tremaining: 230ms\n",
      "61:\tlearn: 0.0705136\ttotal: 364ms\tremaining: 223ms\n",
      "62:\tlearn: 0.0696875\ttotal: 369ms\tremaining: 217ms\n",
      "63:\tlearn: 0.0684278\ttotal: 374ms\tremaining: 211ms\n",
      "64:\tlearn: 0.0676783\ttotal: 379ms\tremaining: 204ms\n",
      "65:\tlearn: 0.0667364\ttotal: 383ms\tremaining: 197ms\n",
      "66:\tlearn: 0.0659132\ttotal: 388ms\tremaining: 191ms\n",
      "67:\tlearn: 0.0653841\ttotal: 392ms\tremaining: 184ms\n",
      "68:\tlearn: 0.0647317\ttotal: 396ms\tremaining: 178ms\n",
      "69:\tlearn: 0.0639145\ttotal: 401ms\tremaining: 172ms\n",
      "70:\tlearn: 0.0633925\ttotal: 405ms\tremaining: 166ms\n",
      "71:\tlearn: 0.0626648\ttotal: 410ms\tremaining: 159ms\n",
      "72:\tlearn: 0.0619501\ttotal: 415ms\tremaining: 153ms\n",
      "73:\tlearn: 0.0616188\ttotal: 419ms\tremaining: 147ms\n",
      "74:\tlearn: 0.0610194\ttotal: 424ms\tremaining: 141ms\n",
      "75:\tlearn: 0.0604733\ttotal: 429ms\tremaining: 135ms\n",
      "76:\tlearn: 0.0600137\ttotal: 433ms\tremaining: 129ms\n",
      "77:\tlearn: 0.0587590\ttotal: 438ms\tremaining: 123ms\n",
      "78:\tlearn: 0.0580928\ttotal: 442ms\tremaining: 118ms\n",
      "79:\tlearn: 0.0575993\ttotal: 447ms\tremaining: 112ms\n",
      "80:\tlearn: 0.0565833\ttotal: 451ms\tremaining: 106ms\n",
      "81:\tlearn: 0.0562656\ttotal: 456ms\tremaining: 100ms\n",
      "82:\tlearn: 0.0559353\ttotal: 460ms\tremaining: 94.2ms\n",
      "83:\tlearn: 0.0554664\ttotal: 464ms\tremaining: 88.4ms\n",
      "84:\tlearn: 0.0548608\ttotal: 469ms\tremaining: 82.7ms\n",
      "85:\tlearn: 0.0539429\ttotal: 474ms\tremaining: 77.1ms\n",
      "86:\tlearn: 0.0536168\ttotal: 478ms\tremaining: 71.4ms\n",
      "87:\tlearn: 0.0533254\ttotal: 483ms\tremaining: 65.8ms\n",
      "88:\tlearn: 0.0529359\ttotal: 487ms\tremaining: 60.2ms\n",
      "89:\tlearn: 0.0524163\ttotal: 491ms\tremaining: 54.6ms\n",
      "90:\tlearn: 0.0519104\ttotal: 496ms\tremaining: 49ms\n",
      "91:\tlearn: 0.0517078\ttotal: 500ms\tremaining: 43.5ms\n",
      "92:\tlearn: 0.0512222\ttotal: 505ms\tremaining: 38ms\n",
      "93:\tlearn: 0.0509407\ttotal: 510ms\tremaining: 32.5ms\n",
      "94:\tlearn: 0.0503620\ttotal: 514ms\tremaining: 27.1ms\n",
      "95:\tlearn: 0.0500811\ttotal: 519ms\tremaining: 21.6ms\n",
      "96:\tlearn: 0.0499203\ttotal: 523ms\tremaining: 16.2ms\n",
      "97:\tlearn: 0.0496955\ttotal: 527ms\tremaining: 10.8ms\n",
      "98:\tlearn: 0.0495331\ttotal: 531ms\tremaining: 5.37ms\n",
      "99:\tlearn: 0.0486919\ttotal: 536ms\tremaining: 0us\n",
      "0:\tlearn: 1.3010145\ttotal: 15ms\tremaining: 1.48s\n",
      "1:\tlearn: 1.0983854\ttotal: 25.2ms\tremaining: 1.24s\n",
      "2:\tlearn: 0.9531997\ttotal: 34.5ms\tremaining: 1.11s\n",
      "3:\tlearn: 0.8344536\ttotal: 43.8ms\tremaining: 1.05s\n",
      "4:\tlearn: 0.7421075\ttotal: 52.1ms\tremaining: 991ms\n",
      "5:\tlearn: 0.6621551\ttotal: 60.3ms\tremaining: 944ms\n",
      "6:\tlearn: 0.5971463\ttotal: 67.5ms\tremaining: 897ms\n",
      "7:\tlearn: 0.5414377\ttotal: 75.2ms\tremaining: 864ms\n",
      "8:\tlearn: 0.4945576\ttotal: 82.3ms\tremaining: 833ms\n",
      "9:\tlearn: 0.4503308\ttotal: 88.8ms\tremaining: 799ms\n",
      "10:\tlearn: 0.4133203\ttotal: 94.9ms\tremaining: 768ms\n",
      "11:\tlearn: 0.3812771\ttotal: 101ms\tremaining: 738ms\n",
      "12:\tlearn: 0.3521971\ttotal: 107ms\tremaining: 714ms\n",
      "13:\tlearn: 0.3257960\ttotal: 112ms\tremaining: 689ms\n",
      "14:\tlearn: 0.3045460\ttotal: 118ms\tremaining: 670ms\n",
      "15:\tlearn: 0.2845031\ttotal: 124ms\tremaining: 651ms\n",
      "16:\tlearn: 0.2660028\ttotal: 130ms\tremaining: 633ms\n",
      "17:\tlearn: 0.2497395\ttotal: 135ms\tremaining: 615ms\n",
      "18:\tlearn: 0.2354636\ttotal: 141ms\tremaining: 599ms\n",
      "19:\tlearn: 0.2219474\ttotal: 146ms\tremaining: 583ms\n",
      "20:\tlearn: 0.2088616\ttotal: 151ms\tremaining: 568ms\n",
      "21:\tlearn: 0.1973058\ttotal: 156ms\tremaining: 554ms\n",
      "22:\tlearn: 0.1868901\ttotal: 161ms\tremaining: 539ms\n",
      "23:\tlearn: 0.1774586\ttotal: 166ms\tremaining: 527ms\n",
      "24:\tlearn: 0.1693798\ttotal: 171ms\tremaining: 514ms\n",
      "25:\tlearn: 0.1616819\ttotal: 176ms\tremaining: 501ms\n",
      "26:\tlearn: 0.1547617\ttotal: 181ms\tremaining: 489ms\n",
      "27:\tlearn: 0.1486697\ttotal: 186ms\tremaining: 478ms\n",
      "28:\tlearn: 0.1427913\ttotal: 190ms\tremaining: 466ms\n",
      "29:\tlearn: 0.1368385\ttotal: 195ms\tremaining: 455ms\n",
      "30:\tlearn: 0.1314748\ttotal: 200ms\tremaining: 444ms\n",
      "31:\tlearn: 0.1275125\ttotal: 204ms\tremaining: 434ms\n",
      "32:\tlearn: 0.1232409\ttotal: 209ms\tremaining: 424ms\n",
      "33:\tlearn: 0.1188678\ttotal: 214ms\tremaining: 415ms\n",
      "34:\tlearn: 0.1151025\ttotal: 219ms\tremaining: 407ms\n",
      "35:\tlearn: 0.1113538\ttotal: 224ms\tremaining: 399ms\n",
      "36:\tlearn: 0.1086687\ttotal: 229ms\tremaining: 390ms\n",
      "37:\tlearn: 0.1062131\ttotal: 234ms\tremaining: 382ms\n",
      "38:\tlearn: 0.1040681\ttotal: 238ms\tremaining: 373ms\n",
      "39:\tlearn: 0.1022913\ttotal: 243ms\tremaining: 364ms\n",
      "40:\tlearn: 0.0994324\ttotal: 248ms\tremaining: 356ms\n",
      "41:\tlearn: 0.0968542\ttotal: 252ms\tremaining: 349ms\n",
      "42:\tlearn: 0.0950294\ttotal: 257ms\tremaining: 341ms\n",
      "43:\tlearn: 0.0930986\ttotal: 262ms\tremaining: 333ms\n",
      "44:\tlearn: 0.0915510\ttotal: 266ms\tremaining: 325ms\n",
      "45:\tlearn: 0.0894104\ttotal: 271ms\tremaining: 318ms\n",
      "46:\tlearn: 0.0873581\ttotal: 275ms\tremaining: 310ms\n",
      "47:\tlearn: 0.0857159\ttotal: 280ms\tremaining: 303ms\n",
      "48:\tlearn: 0.0838057\ttotal: 285ms\tremaining: 296ms\n",
      "49:\tlearn: 0.0813442\ttotal: 290ms\tremaining: 290ms\n",
      "50:\tlearn: 0.0799778\ttotal: 294ms\tremaining: 283ms\n",
      "51:\tlearn: 0.0784017\ttotal: 299ms\tremaining: 276ms\n",
      "52:\tlearn: 0.0766491\ttotal: 305ms\tremaining: 270ms\n",
      "53:\tlearn: 0.0756020\ttotal: 309ms\tremaining: 264ms\n",
      "54:\tlearn: 0.0749504\ttotal: 314ms\tremaining: 257ms\n",
      "55:\tlearn: 0.0741052\ttotal: 319ms\tremaining: 251ms\n",
      "56:\tlearn: 0.0728350\ttotal: 324ms\tremaining: 244ms\n",
      "57:\tlearn: 0.0721601\ttotal: 328ms\tremaining: 237ms\n",
      "58:\tlearn: 0.0709163\ttotal: 333ms\tremaining: 231ms\n",
      "59:\tlearn: 0.0693476\ttotal: 338ms\tremaining: 225ms\n",
      "60:\tlearn: 0.0688687\ttotal: 342ms\tremaining: 219ms\n",
      "61:\tlearn: 0.0680713\ttotal: 346ms\tremaining: 212ms\n",
      "62:\tlearn: 0.0672020\ttotal: 351ms\tremaining: 206ms\n",
      "63:\tlearn: 0.0665549\ttotal: 356ms\tremaining: 200ms\n",
      "64:\tlearn: 0.0659661\ttotal: 360ms\tremaining: 194ms\n",
      "65:\tlearn: 0.0644075\ttotal: 365ms\tremaining: 188ms\n",
      "66:\tlearn: 0.0633466\ttotal: 370ms\tremaining: 182ms\n",
      "67:\tlearn: 0.0628963\ttotal: 375ms\tremaining: 176ms\n",
      "68:\tlearn: 0.0616271\ttotal: 380ms\tremaining: 171ms\n",
      "69:\tlearn: 0.0610140\ttotal: 384ms\tremaining: 165ms\n",
      "70:\tlearn: 0.0604812\ttotal: 389ms\tremaining: 159ms\n",
      "71:\tlearn: 0.0599131\ttotal: 394ms\tremaining: 153ms\n",
      "72:\tlearn: 0.0590663\ttotal: 398ms\tremaining: 147ms\n",
      "73:\tlearn: 0.0586904\ttotal: 402ms\tremaining: 141ms\n",
      "74:\tlearn: 0.0577788\ttotal: 407ms\tremaining: 136ms\n",
      "75:\tlearn: 0.0571169\ttotal: 412ms\tremaining: 130ms\n",
      "76:\tlearn: 0.0568472\ttotal: 417ms\tremaining: 124ms\n",
      "77:\tlearn: 0.0555049\ttotal: 422ms\tremaining: 119ms\n",
      "78:\tlearn: 0.0550891\ttotal: 426ms\tremaining: 113ms\n",
      "79:\tlearn: 0.0547115\ttotal: 434ms\tremaining: 108ms\n",
      "80:\tlearn: 0.0544165\ttotal: 438ms\tremaining: 103ms\n",
      "81:\tlearn: 0.0539319\ttotal: 443ms\tremaining: 97.2ms\n",
      "82:\tlearn: 0.0531390\ttotal: 447ms\tremaining: 91.7ms\n",
      "83:\tlearn: 0.0526751\ttotal: 452ms\tremaining: 86ms\n",
      "84:\tlearn: 0.0521975\ttotal: 456ms\tremaining: 80.5ms\n",
      "85:\tlearn: 0.0515096\ttotal: 461ms\tremaining: 75ms\n",
      "86:\tlearn: 0.0508276\ttotal: 466ms\tremaining: 69.6ms\n",
      "87:\tlearn: 0.0506122\ttotal: 470ms\tremaining: 64.1ms\n",
      "88:\tlearn: 0.0503345\ttotal: 474ms\tremaining: 58.6ms\n",
      "89:\tlearn: 0.0500358\ttotal: 479ms\tremaining: 53.2ms\n",
      "90:\tlearn: 0.0496892\ttotal: 483ms\tremaining: 47.8ms\n",
      "91:\tlearn: 0.0494736\ttotal: 487ms\tremaining: 42.3ms\n",
      "92:\tlearn: 0.0491436\ttotal: 492ms\tremaining: 37ms\n",
      "93:\tlearn: 0.0486658\ttotal: 497ms\tremaining: 31.7ms\n",
      "94:\tlearn: 0.0483947\ttotal: 501ms\tremaining: 26.4ms\n",
      "95:\tlearn: 0.0475643\ttotal: 506ms\tremaining: 21.1ms\n",
      "96:\tlearn: 0.0471940\ttotal: 511ms\tremaining: 15.8ms\n",
      "97:\tlearn: 0.0467765\ttotal: 515ms\tremaining: 10.5ms\n",
      "98:\tlearn: 0.0463522\ttotal: 520ms\tremaining: 5.25ms\n",
      "99:\tlearn: 0.0459028\ttotal: 525ms\tremaining: 0us\n",
      "0:\tlearn: 1.3041571\ttotal: 13.1ms\tremaining: 1.3s\n",
      "1:\tlearn: 1.1027479\ttotal: 24.4ms\tremaining: 1.19s\n",
      "2:\tlearn: 0.9574436\ttotal: 34.6ms\tremaining: 1.12s\n",
      "3:\tlearn: 0.8383555\ttotal: 44.5ms\tremaining: 1.07s\n",
      "4:\tlearn: 0.7458686\ttotal: 54ms\tremaining: 1.02s\n",
      "5:\tlearn: 0.6671569\ttotal: 63.2ms\tremaining: 990ms\n",
      "6:\tlearn: 0.6018440\ttotal: 71.1ms\tremaining: 945ms\n",
      "7:\tlearn: 0.5440443\ttotal: 79ms\tremaining: 909ms\n",
      "8:\tlearn: 0.4970626\ttotal: 86.3ms\tremaining: 873ms\n",
      "9:\tlearn: 0.4526593\ttotal: 93.1ms\tremaining: 838ms\n",
      "10:\tlearn: 0.4157364\ttotal: 99.6ms\tremaining: 806ms\n",
      "11:\tlearn: 0.3843079\ttotal: 105ms\tremaining: 771ms\n",
      "12:\tlearn: 0.3546416\ttotal: 112ms\tremaining: 747ms\n",
      "13:\tlearn: 0.3296768\ttotal: 117ms\tremaining: 718ms\n",
      "14:\tlearn: 0.3085579\ttotal: 122ms\tremaining: 691ms\n",
      "15:\tlearn: 0.2893311\ttotal: 127ms\tremaining: 666ms\n",
      "16:\tlearn: 0.2703065\ttotal: 132ms\tremaining: 644ms\n",
      "17:\tlearn: 0.2541168\ttotal: 137ms\tremaining: 622ms\n",
      "18:\tlearn: 0.2398206\ttotal: 141ms\tremaining: 603ms\n",
      "19:\tlearn: 0.2262294\ttotal: 146ms\tremaining: 586ms\n",
      "20:\tlearn: 0.2139332\ttotal: 151ms\tremaining: 569ms\n",
      "21:\tlearn: 0.2026040\ttotal: 157ms\tremaining: 555ms\n",
      "22:\tlearn: 0.1919059\ttotal: 162ms\tremaining: 541ms\n",
      "23:\tlearn: 0.1825519\ttotal: 166ms\tremaining: 527ms\n",
      "24:\tlearn: 0.1744532\ttotal: 171ms\tremaining: 514ms\n",
      "25:\tlearn: 0.1666279\ttotal: 176ms\tremaining: 501ms\n",
      "26:\tlearn: 0.1589118\ttotal: 181ms\tremaining: 489ms\n",
      "27:\tlearn: 0.1525324\ttotal: 185ms\tremaining: 476ms\n",
      "28:\tlearn: 0.1457299\ttotal: 190ms\tremaining: 466ms\n",
      "29:\tlearn: 0.1394577\ttotal: 196ms\tremaining: 457ms\n",
      "30:\tlearn: 0.1342483\ttotal: 201ms\tremaining: 447ms\n",
      "31:\tlearn: 0.1297831\ttotal: 206ms\tremaining: 438ms\n",
      "32:\tlearn: 0.1257332\ttotal: 211ms\tremaining: 428ms\n",
      "33:\tlearn: 0.1216626\ttotal: 216ms\tremaining: 418ms\n",
      "34:\tlearn: 0.1185692\ttotal: 220ms\tremaining: 409ms\n",
      "35:\tlearn: 0.1146247\ttotal: 225ms\tremaining: 400ms\n",
      "36:\tlearn: 0.1120849\ttotal: 229ms\tremaining: 390ms\n",
      "37:\tlearn: 0.1080596\ttotal: 234ms\tremaining: 382ms\n",
      "38:\tlearn: 0.1049854\ttotal: 239ms\tremaining: 373ms\n",
      "39:\tlearn: 0.1027907\ttotal: 243ms\tremaining: 365ms\n",
      "40:\tlearn: 0.1011389\ttotal: 273ms\tremaining: 392ms\n",
      "41:\tlearn: 0.0987187\ttotal: 277ms\tremaining: 383ms\n",
      "42:\tlearn: 0.0966502\ttotal: 281ms\tremaining: 373ms\n",
      "43:\tlearn: 0.0943539\ttotal: 286ms\tremaining: 364ms\n",
      "44:\tlearn: 0.0923685\ttotal: 290ms\tremaining: 354ms\n",
      "45:\tlearn: 0.0904158\ttotal: 294ms\tremaining: 346ms\n",
      "46:\tlearn: 0.0892238\ttotal: 298ms\tremaining: 337ms\n",
      "47:\tlearn: 0.0883296\ttotal: 303ms\tremaining: 328ms\n",
      "48:\tlearn: 0.0874642\ttotal: 307ms\tremaining: 319ms\n",
      "49:\tlearn: 0.0852269\ttotal: 312ms\tremaining: 312ms\n",
      "50:\tlearn: 0.0841319\ttotal: 316ms\tremaining: 303ms\n",
      "51:\tlearn: 0.0817221\ttotal: 320ms\tremaining: 296ms\n",
      "52:\tlearn: 0.0795668\ttotal: 325ms\tremaining: 288ms\n",
      "53:\tlearn: 0.0783639\ttotal: 329ms\tremaining: 281ms\n",
      "54:\tlearn: 0.0770442\ttotal: 334ms\tremaining: 273ms\n",
      "55:\tlearn: 0.0762317\ttotal: 338ms\tremaining: 266ms\n",
      "56:\tlearn: 0.0752466\ttotal: 342ms\tremaining: 258ms\n",
      "57:\tlearn: 0.0744363\ttotal: 347ms\tremaining: 251ms\n",
      "58:\tlearn: 0.0739355\ttotal: 351ms\tremaining: 244ms\n",
      "59:\tlearn: 0.0724901\ttotal: 356ms\tremaining: 237ms\n",
      "60:\tlearn: 0.0717257\ttotal: 360ms\tremaining: 230ms\n",
      "61:\tlearn: 0.0706138\ttotal: 365ms\tremaining: 223ms\n",
      "62:\tlearn: 0.0697913\ttotal: 369ms\tremaining: 217ms\n",
      "63:\tlearn: 0.0690932\ttotal: 374ms\tremaining: 210ms\n",
      "64:\tlearn: 0.0686367\ttotal: 378ms\tremaining: 204ms\n",
      "65:\tlearn: 0.0680405\ttotal: 382ms\tremaining: 197ms\n",
      "66:\tlearn: 0.0670099\ttotal: 387ms\tremaining: 191ms\n",
      "67:\tlearn: 0.0664736\ttotal: 392ms\tremaining: 185ms\n",
      "68:\tlearn: 0.0654998\ttotal: 398ms\tremaining: 179ms\n",
      "69:\tlearn: 0.0644778\ttotal: 402ms\tremaining: 172ms\n",
      "70:\tlearn: 0.0639035\ttotal: 407ms\tremaining: 166ms\n",
      "71:\tlearn: 0.0633781\ttotal: 411ms\tremaining: 160ms\n",
      "72:\tlearn: 0.0626689\ttotal: 416ms\tremaining: 154ms\n",
      "73:\tlearn: 0.0617956\ttotal: 420ms\tremaining: 148ms\n",
      "74:\tlearn: 0.0609978\ttotal: 425ms\tremaining: 142ms\n",
      "75:\tlearn: 0.0602228\ttotal: 429ms\tremaining: 136ms\n",
      "76:\tlearn: 0.0598297\ttotal: 434ms\tremaining: 129ms\n",
      "77:\tlearn: 0.0594685\ttotal: 438ms\tremaining: 124ms\n",
      "78:\tlearn: 0.0588577\ttotal: 443ms\tremaining: 118ms\n",
      "79:\tlearn: 0.0581191\ttotal: 447ms\tremaining: 112ms\n",
      "80:\tlearn: 0.0571588\ttotal: 452ms\tremaining: 106ms\n",
      "81:\tlearn: 0.0567358\ttotal: 457ms\tremaining: 100ms\n",
      "82:\tlearn: 0.0564343\ttotal: 461ms\tremaining: 94.5ms\n",
      "83:\tlearn: 0.0561003\ttotal: 466ms\tremaining: 88.7ms\n",
      "84:\tlearn: 0.0559239\ttotal: 470ms\tremaining: 82.9ms\n",
      "85:\tlearn: 0.0556150\ttotal: 474ms\tremaining: 77.2ms\n",
      "86:\tlearn: 0.0553226\ttotal: 479ms\tremaining: 71.5ms\n",
      "87:\tlearn: 0.0551780\ttotal: 483ms\tremaining: 65.8ms\n",
      "88:\tlearn: 0.0539685\ttotal: 487ms\tremaining: 60.2ms\n",
      "89:\tlearn: 0.0537415\ttotal: 491ms\tremaining: 54.6ms\n",
      "90:\tlearn: 0.0534366\ttotal: 496ms\tremaining: 49ms\n",
      "91:\tlearn: 0.0529066\ttotal: 500ms\tremaining: 43.5ms\n",
      "92:\tlearn: 0.0525259\ttotal: 505ms\tremaining: 38ms\n",
      "93:\tlearn: 0.0521253\ttotal: 510ms\tremaining: 32.5ms\n",
      "94:\tlearn: 0.0518624\ttotal: 514ms\tremaining: 27.1ms\n",
      "95:\tlearn: 0.0515300\ttotal: 519ms\tremaining: 21.6ms\n",
      "96:\tlearn: 0.0511250\ttotal: 523ms\tremaining: 16.2ms\n",
      "97:\tlearn: 0.0506894\ttotal: 528ms\tremaining: 10.8ms\n",
      "98:\tlearn: 0.0499204\ttotal: 533ms\tremaining: 5.38ms\n",
      "99:\tlearn: 0.0496695\ttotal: 537ms\tremaining: 0us\n",
      "0:\tlearn: 1.3027906\ttotal: 12.4ms\tremaining: 1.23s\n",
      "1:\tlearn: 1.1029063\ttotal: 24ms\tremaining: 1.18s\n",
      "2:\tlearn: 0.9553906\ttotal: 34.3ms\tremaining: 1.11s\n",
      "3:\tlearn: 0.8365457\ttotal: 43.5ms\tremaining: 1.04s\n",
      "4:\tlearn: 0.7422359\ttotal: 52.6ms\tremaining: 1000ms\n",
      "5:\tlearn: 0.6640459\ttotal: 61.7ms\tremaining: 967ms\n",
      "6:\tlearn: 0.5981979\ttotal: 70.1ms\tremaining: 931ms\n",
      "7:\tlearn: 0.5405973\ttotal: 78.6ms\tremaining: 904ms\n",
      "8:\tlearn: 0.4938136\ttotal: 86.5ms\tremaining: 875ms\n",
      "9:\tlearn: 0.4496071\ttotal: 93.6ms\tremaining: 842ms\n",
      "10:\tlearn: 0.4135613\ttotal: 100ms\tremaining: 813ms\n",
      "11:\tlearn: 0.3824964\ttotal: 107ms\tremaining: 782ms\n",
      "12:\tlearn: 0.3527307\ttotal: 113ms\tremaining: 756ms\n",
      "13:\tlearn: 0.3281655\ttotal: 119ms\tremaining: 729ms\n",
      "14:\tlearn: 0.3067012\ttotal: 124ms\tremaining: 705ms\n",
      "15:\tlearn: 0.2876612\ttotal: 130ms\tremaining: 682ms\n",
      "16:\tlearn: 0.2697509\ttotal: 136ms\tremaining: 662ms\n",
      "17:\tlearn: 0.2536214\ttotal: 141ms\tremaining: 643ms\n",
      "18:\tlearn: 0.2389942\ttotal: 147ms\tremaining: 625ms\n",
      "19:\tlearn: 0.2253094\ttotal: 152ms\tremaining: 608ms\n",
      "20:\tlearn: 0.2119882\ttotal: 157ms\tremaining: 591ms\n",
      "21:\tlearn: 0.2006869\ttotal: 162ms\tremaining: 575ms\n",
      "22:\tlearn: 0.1901679\ttotal: 167ms\tremaining: 559ms\n",
      "23:\tlearn: 0.1805364\ttotal: 172ms\tremaining: 544ms\n",
      "24:\tlearn: 0.1717004\ttotal: 177ms\tremaining: 530ms\n",
      "25:\tlearn: 0.1642603\ttotal: 181ms\tremaining: 516ms\n",
      "26:\tlearn: 0.1575428\ttotal: 186ms\tremaining: 503ms\n",
      "27:\tlearn: 0.1508670\ttotal: 191ms\tremaining: 491ms\n",
      "28:\tlearn: 0.1441781\ttotal: 196ms\tremaining: 479ms\n",
      "29:\tlearn: 0.1388294\ttotal: 200ms\tremaining: 468ms\n",
      "30:\tlearn: 0.1331390\ttotal: 205ms\tremaining: 457ms\n",
      "31:\tlearn: 0.1289968\ttotal: 210ms\tremaining: 447ms\n",
      "32:\tlearn: 0.1241815\ttotal: 215ms\tremaining: 436ms\n",
      "33:\tlearn: 0.1201084\ttotal: 219ms\tremaining: 426ms\n",
      "34:\tlearn: 0.1154995\ttotal: 225ms\tremaining: 418ms\n",
      "35:\tlearn: 0.1113483\ttotal: 230ms\tremaining: 408ms\n",
      "36:\tlearn: 0.1086470\ttotal: 234ms\tremaining: 398ms\n",
      "37:\tlearn: 0.1050499\ttotal: 239ms\tremaining: 389ms\n",
      "38:\tlearn: 0.1028733\ttotal: 243ms\tremaining: 380ms\n",
      "39:\tlearn: 0.1002224\ttotal: 247ms\tremaining: 371ms\n",
      "40:\tlearn: 0.0969289\ttotal: 252ms\tremaining: 363ms\n",
      "41:\tlearn: 0.0945465\ttotal: 257ms\tremaining: 354ms\n",
      "42:\tlearn: 0.0930807\ttotal: 261ms\tremaining: 346ms\n",
      "43:\tlearn: 0.0905974\ttotal: 266ms\tremaining: 339ms\n",
      "44:\tlearn: 0.0883704\ttotal: 270ms\tremaining: 331ms\n",
      "45:\tlearn: 0.0865436\ttotal: 275ms\tremaining: 323ms\n",
      "46:\tlearn: 0.0845972\ttotal: 280ms\tremaining: 315ms\n",
      "47:\tlearn: 0.0838062\ttotal: 284ms\tremaining: 308ms\n",
      "48:\tlearn: 0.0812575\ttotal: 289ms\tremaining: 301ms\n",
      "49:\tlearn: 0.0796854\ttotal: 293ms\tremaining: 293ms\n",
      "50:\tlearn: 0.0787330\ttotal: 298ms\tremaining: 286ms\n",
      "51:\tlearn: 0.0774753\ttotal: 302ms\tremaining: 279ms\n",
      "52:\tlearn: 0.0754945\ttotal: 307ms\tremaining: 272ms\n",
      "53:\tlearn: 0.0744228\ttotal: 312ms\tremaining: 266ms\n",
      "54:\tlearn: 0.0733891\ttotal: 316ms\tremaining: 259ms\n",
      "55:\tlearn: 0.0725788\ttotal: 321ms\tremaining: 252ms\n",
      "56:\tlearn: 0.0716868\ttotal: 325ms\tremaining: 245ms\n",
      "57:\tlearn: 0.0705099\ttotal: 330ms\tremaining: 239ms\n",
      "58:\tlearn: 0.0696810\ttotal: 334ms\tremaining: 232ms\n",
      "59:\tlearn: 0.0679434\ttotal: 339ms\tremaining: 226ms\n",
      "60:\tlearn: 0.0670737\ttotal: 343ms\tremaining: 220ms\n",
      "61:\tlearn: 0.0660409\ttotal: 348ms\tremaining: 214ms\n",
      "62:\tlearn: 0.0653418\ttotal: 353ms\tremaining: 207ms\n",
      "63:\tlearn: 0.0648435\ttotal: 357ms\tremaining: 201ms\n",
      "64:\tlearn: 0.0642655\ttotal: 361ms\tremaining: 194ms\n",
      "65:\tlearn: 0.0632265\ttotal: 366ms\tremaining: 188ms\n",
      "66:\tlearn: 0.0618548\ttotal: 371ms\tremaining: 182ms\n",
      "67:\tlearn: 0.0613523\ttotal: 375ms\tremaining: 177ms\n",
      "68:\tlearn: 0.0609439\ttotal: 380ms\tremaining: 171ms\n",
      "69:\tlearn: 0.0601657\ttotal: 384ms\tremaining: 164ms\n",
      "70:\tlearn: 0.0598262\ttotal: 388ms\tremaining: 158ms\n",
      "71:\tlearn: 0.0585652\ttotal: 393ms\tremaining: 153ms\n",
      "72:\tlearn: 0.0571328\ttotal: 398ms\tremaining: 147ms\n",
      "73:\tlearn: 0.0567863\ttotal: 402ms\tremaining: 141ms\n",
      "74:\tlearn: 0.0563970\ttotal: 407ms\tremaining: 136ms\n",
      "75:\tlearn: 0.0560922\ttotal: 411ms\tremaining: 130ms\n",
      "76:\tlearn: 0.0556515\ttotal: 416ms\tremaining: 124ms\n",
      "77:\tlearn: 0.0552262\ttotal: 420ms\tremaining: 119ms\n",
      "78:\tlearn: 0.0541565\ttotal: 425ms\tremaining: 113ms\n",
      "79:\tlearn: 0.0537182\ttotal: 430ms\tremaining: 107ms\n",
      "80:\tlearn: 0.0529937\ttotal: 434ms\tremaining: 102ms\n",
      "81:\tlearn: 0.0524224\ttotal: 439ms\tremaining: 96.4ms\n",
      "82:\tlearn: 0.0521328\ttotal: 444ms\tremaining: 90.9ms\n",
      "83:\tlearn: 0.0519107\ttotal: 448ms\tremaining: 85.4ms\n",
      "84:\tlearn: 0.0514884\ttotal: 453ms\tremaining: 79.9ms\n",
      "85:\tlearn: 0.0503957\ttotal: 458ms\tremaining: 74.5ms\n",
      "86:\tlearn: 0.0500967\ttotal: 462ms\tremaining: 69.1ms\n",
      "87:\tlearn: 0.0495715\ttotal: 467ms\tremaining: 63.6ms\n",
      "88:\tlearn: 0.0492403\ttotal: 471ms\tremaining: 58.2ms\n",
      "89:\tlearn: 0.0487048\ttotal: 475ms\tremaining: 52.8ms\n",
      "90:\tlearn: 0.0481859\ttotal: 480ms\tremaining: 47.4ms\n",
      "91:\tlearn: 0.0480040\ttotal: 484ms\tremaining: 42.1ms\n",
      "92:\tlearn: 0.0475057\ttotal: 488ms\tremaining: 36.8ms\n",
      "93:\tlearn: 0.0473426\ttotal: 493ms\tremaining: 31.4ms\n",
      "94:\tlearn: 0.0471158\ttotal: 497ms\tremaining: 26.2ms\n",
      "95:\tlearn: 0.0467765\ttotal: 501ms\tremaining: 20.9ms\n",
      "96:\tlearn: 0.0464959\ttotal: 506ms\tremaining: 15.6ms\n",
      "97:\tlearn: 0.0463018\ttotal: 510ms\tremaining: 10.4ms\n",
      "98:\tlearn: 0.0459777\ttotal: 515ms\tremaining: 5.2ms\n",
      "99:\tlearn: 0.0458544\ttotal: 519ms\tremaining: 0us\n",
      "0:\tlearn: 1.3060084\ttotal: 13ms\tremaining: 1.29s\n",
      "1:\tlearn: 1.1052704\ttotal: 23.9ms\tremaining: 1.17s\n",
      "2:\tlearn: 0.9623351\ttotal: 34.1ms\tremaining: 1.1s\n",
      "3:\tlearn: 0.8430205\ttotal: 43.5ms\tremaining: 1.04s\n",
      "4:\tlearn: 0.7516277\ttotal: 52.3ms\tremaining: 993ms\n",
      "5:\tlearn: 0.6732890\ttotal: 60.9ms\tremaining: 954ms\n",
      "6:\tlearn: 0.6082164\ttotal: 69ms\tremaining: 917ms\n",
      "7:\tlearn: 0.5545793\ttotal: 76.8ms\tremaining: 883ms\n",
      "8:\tlearn: 0.5085769\ttotal: 84ms\tremaining: 849ms\n",
      "9:\tlearn: 0.4632058\ttotal: 91.4ms\tremaining: 822ms\n",
      "10:\tlearn: 0.4259551\ttotal: 98.3ms\tremaining: 795ms\n",
      "11:\tlearn: 0.3913927\ttotal: 105ms\tremaining: 772ms\n",
      "12:\tlearn: 0.3611420\ttotal: 111ms\tremaining: 745ms\n",
      "13:\tlearn: 0.3350309\ttotal: 117ms\tremaining: 721ms\n",
      "14:\tlearn: 0.3132411\ttotal: 124ms\tremaining: 703ms\n",
      "15:\tlearn: 0.2929713\ttotal: 130ms\tremaining: 682ms\n",
      "16:\tlearn: 0.2756646\ttotal: 136ms\tremaining: 662ms\n",
      "17:\tlearn: 0.2591384\ttotal: 141ms\tremaining: 642ms\n",
      "18:\tlearn: 0.2443772\ttotal: 146ms\tremaining: 624ms\n",
      "19:\tlearn: 0.2311445\ttotal: 152ms\tremaining: 608ms\n",
      "20:\tlearn: 0.2182837\ttotal: 157ms\tremaining: 591ms\n",
      "21:\tlearn: 0.2058011\ttotal: 162ms\tremaining: 575ms\n",
      "22:\tlearn: 0.1954440\ttotal: 167ms\tremaining: 559ms\n",
      "23:\tlearn: 0.1858005\ttotal: 172ms\tremaining: 544ms\n",
      "24:\tlearn: 0.1769860\ttotal: 177ms\tremaining: 530ms\n",
      "25:\tlearn: 0.1690236\ttotal: 182ms\tremaining: 517ms\n",
      "26:\tlearn: 0.1623269\ttotal: 187ms\tremaining: 505ms\n",
      "27:\tlearn: 0.1559001\ttotal: 191ms\tremaining: 492ms\n",
      "28:\tlearn: 0.1500533\ttotal: 196ms\tremaining: 481ms\n",
      "29:\tlearn: 0.1448007\ttotal: 201ms\tremaining: 469ms\n",
      "30:\tlearn: 0.1399285\ttotal: 206ms\tremaining: 458ms\n",
      "31:\tlearn: 0.1348039\ttotal: 210ms\tremaining: 447ms\n",
      "32:\tlearn: 0.1298989\ttotal: 215ms\tremaining: 437ms\n",
      "33:\tlearn: 0.1255673\ttotal: 220ms\tremaining: 427ms\n",
      "34:\tlearn: 0.1224588\ttotal: 224ms\tremaining: 417ms\n",
      "35:\tlearn: 0.1182540\ttotal: 229ms\tremaining: 408ms\n",
      "36:\tlearn: 0.1153870\ttotal: 234ms\tremaining: 399ms\n",
      "37:\tlearn: 0.1112753\ttotal: 239ms\tremaining: 389ms\n",
      "38:\tlearn: 0.1081135\ttotal: 243ms\tremaining: 380ms\n",
      "39:\tlearn: 0.1058152\ttotal: 248ms\tremaining: 371ms\n",
      "40:\tlearn: 0.1040320\ttotal: 252ms\tremaining: 363ms\n",
      "41:\tlearn: 0.1018336\ttotal: 257ms\tremaining: 354ms\n",
      "42:\tlearn: 0.0992201\ttotal: 261ms\tremaining: 346ms\n",
      "43:\tlearn: 0.0967704\ttotal: 266ms\tremaining: 338ms\n",
      "44:\tlearn: 0.0951396\ttotal: 270ms\tremaining: 330ms\n",
      "45:\tlearn: 0.0921428\ttotal: 275ms\tremaining: 323ms\n",
      "46:\tlearn: 0.0900052\ttotal: 280ms\tremaining: 316ms\n",
      "47:\tlearn: 0.0883170\ttotal: 284ms\tremaining: 308ms\n",
      "48:\tlearn: 0.0858042\ttotal: 289ms\tremaining: 301ms\n",
      "49:\tlearn: 0.0848921\ttotal: 294ms\tremaining: 294ms\n",
      "50:\tlearn: 0.0834451\ttotal: 299ms\tremaining: 287ms\n",
      "51:\tlearn: 0.0816199\ttotal: 304ms\tremaining: 281ms\n",
      "52:\tlearn: 0.0806640\ttotal: 309ms\tremaining: 274ms\n",
      "53:\tlearn: 0.0795506\ttotal: 313ms\tremaining: 267ms\n",
      "54:\tlearn: 0.0783823\ttotal: 318ms\tremaining: 260ms\n",
      "55:\tlearn: 0.0778565\ttotal: 322ms\tremaining: 253ms\n",
      "56:\tlearn: 0.0768358\ttotal: 327ms\tremaining: 246ms\n",
      "57:\tlearn: 0.0759147\ttotal: 331ms\tremaining: 240ms\n",
      "58:\tlearn: 0.0741788\ttotal: 336ms\tremaining: 234ms\n",
      "59:\tlearn: 0.0722674\ttotal: 341ms\tremaining: 227ms\n",
      "60:\tlearn: 0.0712642\ttotal: 345ms\tremaining: 221ms\n",
      "61:\tlearn: 0.0702549\ttotal: 350ms\tremaining: 214ms\n",
      "62:\tlearn: 0.0694764\ttotal: 354ms\tremaining: 208ms\n",
      "63:\tlearn: 0.0687209\ttotal: 359ms\tremaining: 202ms\n",
      "64:\tlearn: 0.0674523\ttotal: 364ms\tremaining: 196ms\n",
      "65:\tlearn: 0.0670088\ttotal: 369ms\tremaining: 190ms\n",
      "66:\tlearn: 0.0665667\ttotal: 373ms\tremaining: 184ms\n",
      "67:\tlearn: 0.0659493\ttotal: 377ms\tremaining: 177ms\n",
      "68:\tlearn: 0.0649222\ttotal: 382ms\tremaining: 172ms\n",
      "69:\tlearn: 0.0644394\ttotal: 387ms\tremaining: 166ms\n",
      "70:\tlearn: 0.0631767\ttotal: 392ms\tremaining: 160ms\n",
      "71:\tlearn: 0.0627336\ttotal: 397ms\tremaining: 154ms\n",
      "72:\tlearn: 0.0620688\ttotal: 401ms\tremaining: 148ms\n",
      "73:\tlearn: 0.0617149\ttotal: 406ms\tremaining: 143ms\n",
      "74:\tlearn: 0.0608527\ttotal: 411ms\tremaining: 137ms\n",
      "75:\tlearn: 0.0600880\ttotal: 416ms\tremaining: 131ms\n",
      "76:\tlearn: 0.0597288\ttotal: 420ms\tremaining: 126ms\n",
      "77:\tlearn: 0.0584614\ttotal: 425ms\tremaining: 120ms\n",
      "78:\tlearn: 0.0579292\ttotal: 430ms\tremaining: 114ms\n",
      "79:\tlearn: 0.0572613\ttotal: 434ms\tremaining: 109ms\n",
      "80:\tlearn: 0.0568099\ttotal: 439ms\tremaining: 103ms\n",
      "81:\tlearn: 0.0563647\ttotal: 444ms\tremaining: 97.5ms\n",
      "82:\tlearn: 0.0557470\ttotal: 449ms\tremaining: 91.9ms\n",
      "83:\tlearn: 0.0553100\ttotal: 453ms\tremaining: 86.3ms\n",
      "84:\tlearn: 0.0550164\ttotal: 457ms\tremaining: 80.7ms\n",
      "85:\tlearn: 0.0542141\ttotal: 462ms\tremaining: 75.2ms\n",
      "86:\tlearn: 0.0540160\ttotal: 467ms\tremaining: 69.7ms\n",
      "87:\tlearn: 0.0536338\ttotal: 471ms\tremaining: 64.2ms\n",
      "88:\tlearn: 0.0532314\ttotal: 475ms\tremaining: 58.7ms\n",
      "89:\tlearn: 0.0526344\ttotal: 480ms\tremaining: 53.3ms\n",
      "90:\tlearn: 0.0520114\ttotal: 485ms\tremaining: 47.9ms\n",
      "91:\tlearn: 0.0518326\ttotal: 489ms\tremaining: 42.5ms\n",
      "92:\tlearn: 0.0513749\ttotal: 494ms\tremaining: 37.2ms\n",
      "93:\tlearn: 0.0507176\ttotal: 499ms\tremaining: 31.8ms\n",
      "94:\tlearn: 0.0504258\ttotal: 503ms\tremaining: 26.5ms\n",
      "95:\tlearn: 0.0500654\ttotal: 507ms\tremaining: 21.1ms\n",
      "96:\tlearn: 0.0498119\ttotal: 512ms\tremaining: 15.8ms\n",
      "97:\tlearn: 0.0492855\ttotal: 517ms\tremaining: 10.5ms\n",
      "98:\tlearn: 0.0487177\ttotal: 521ms\tremaining: 5.27ms\n",
      "99:\tlearn: 0.0485770\ttotal: 526ms\tremaining: 0us\n",
      "0:\tlearn: 1.3049885\ttotal: 13.5ms\tremaining: 1.34s\n",
      "1:\tlearn: 1.1026756\ttotal: 23.2ms\tremaining: 1.14s\n",
      "2:\tlearn: 0.9573914\ttotal: 32.2ms\tremaining: 1.04s\n",
      "3:\tlearn: 0.8386398\ttotal: 41ms\tremaining: 984ms\n",
      "4:\tlearn: 0.7453294\ttotal: 49.3ms\tremaining: 936ms\n",
      "5:\tlearn: 0.6675119\ttotal: 57.5ms\tremaining: 901ms\n",
      "6:\tlearn: 0.6023684\ttotal: 65.6ms\tremaining: 872ms\n",
      "7:\tlearn: 0.5451763\ttotal: 73ms\tremaining: 840ms\n",
      "8:\tlearn: 0.5001331\ttotal: 80.1ms\tremaining: 809ms\n",
      "9:\tlearn: 0.4591111\ttotal: 86.4ms\tremaining: 778ms\n",
      "10:\tlearn: 0.4218071\ttotal: 92.8ms\tremaining: 750ms\n",
      "11:\tlearn: 0.3903864\ttotal: 98.8ms\tremaining: 725ms\n",
      "12:\tlearn: 0.3607894\ttotal: 105ms\tremaining: 706ms\n",
      "13:\tlearn: 0.3338571\ttotal: 111ms\tremaining: 684ms\n",
      "14:\tlearn: 0.3122454\ttotal: 118ms\tremaining: 667ms\n",
      "15:\tlearn: 0.2920019\ttotal: 124ms\tremaining: 649ms\n",
      "16:\tlearn: 0.2733442\ttotal: 129ms\tremaining: 630ms\n",
      "17:\tlearn: 0.2566984\ttotal: 134ms\tremaining: 612ms\n",
      "18:\tlearn: 0.2412805\ttotal: 140ms\tremaining: 595ms\n",
      "19:\tlearn: 0.2279548\ttotal: 145ms\tremaining: 580ms\n",
      "20:\tlearn: 0.2161157\ttotal: 150ms\tremaining: 565ms\n",
      "21:\tlearn: 0.2042804\ttotal: 155ms\tremaining: 550ms\n",
      "22:\tlearn: 0.1942101\ttotal: 160ms\tremaining: 535ms\n",
      "23:\tlearn: 0.1852594\ttotal: 165ms\tremaining: 521ms\n",
      "24:\tlearn: 0.1760346\ttotal: 169ms\tremaining: 508ms\n",
      "25:\tlearn: 0.1690884\ttotal: 174ms\tremaining: 496ms\n",
      "26:\tlearn: 0.1614484\ttotal: 179ms\tremaining: 484ms\n",
      "27:\tlearn: 0.1548320\ttotal: 184ms\tremaining: 473ms\n",
      "28:\tlearn: 0.1485737\ttotal: 189ms\tremaining: 463ms\n",
      "29:\tlearn: 0.1426371\ttotal: 194ms\tremaining: 454ms\n",
      "30:\tlearn: 0.1370234\ttotal: 200ms\tremaining: 445ms\n",
      "31:\tlearn: 0.1324482\ttotal: 205ms\tremaining: 436ms\n",
      "32:\tlearn: 0.1278522\ttotal: 211ms\tremaining: 428ms\n",
      "33:\tlearn: 0.1244017\ttotal: 216ms\tremaining: 419ms\n",
      "34:\tlearn: 0.1212880\ttotal: 221ms\tremaining: 411ms\n",
      "35:\tlearn: 0.1174994\ttotal: 226ms\tremaining: 402ms\n",
      "36:\tlearn: 0.1139815\ttotal: 231ms\tremaining: 393ms\n",
      "37:\tlearn: 0.1117804\ttotal: 236ms\tremaining: 385ms\n",
      "38:\tlearn: 0.1094657\ttotal: 240ms\tremaining: 375ms\n",
      "39:\tlearn: 0.1076005\ttotal: 244ms\tremaining: 367ms\n",
      "40:\tlearn: 0.1039243\ttotal: 249ms\tremaining: 359ms\n",
      "41:\tlearn: 0.1020547\ttotal: 253ms\tremaining: 350ms\n",
      "42:\tlearn: 0.1002961\ttotal: 258ms\tremaining: 342ms\n",
      "43:\tlearn: 0.0977609\ttotal: 263ms\tremaining: 334ms\n",
      "44:\tlearn: 0.0966401\ttotal: 267ms\tremaining: 326ms\n",
      "45:\tlearn: 0.0946752\ttotal: 272ms\tremaining: 319ms\n",
      "46:\tlearn: 0.0925269\ttotal: 276ms\tremaining: 311ms\n",
      "47:\tlearn: 0.0916400\ttotal: 281ms\tremaining: 304ms\n",
      "48:\tlearn: 0.0888536\ttotal: 287ms\tremaining: 298ms\n",
      "49:\tlearn: 0.0870866\ttotal: 291ms\tremaining: 291ms\n",
      "50:\tlearn: 0.0858299\ttotal: 296ms\tremaining: 284ms\n",
      "51:\tlearn: 0.0831670\ttotal: 301ms\tremaining: 278ms\n",
      "52:\tlearn: 0.0807909\ttotal: 306ms\tremaining: 272ms\n",
      "53:\tlearn: 0.0796914\ttotal: 311ms\tremaining: 265ms\n",
      "54:\tlearn: 0.0785799\ttotal: 315ms\tremaining: 258ms\n",
      "55:\tlearn: 0.0780780\ttotal: 320ms\tremaining: 251ms\n",
      "56:\tlearn: 0.0770034\ttotal: 324ms\tremaining: 244ms\n",
      "57:\tlearn: 0.0764568\ttotal: 328ms\tremaining: 238ms\n",
      "58:\tlearn: 0.0753022\ttotal: 333ms\tremaining: 231ms\n",
      "59:\tlearn: 0.0735847\ttotal: 337ms\tremaining: 225ms\n",
      "60:\tlearn: 0.0727493\ttotal: 342ms\tremaining: 218ms\n",
      "61:\tlearn: 0.0710077\ttotal: 347ms\tremaining: 213ms\n",
      "62:\tlearn: 0.0702088\ttotal: 352ms\tremaining: 206ms\n",
      "63:\tlearn: 0.0692264\ttotal: 356ms\tremaining: 200ms\n",
      "64:\tlearn: 0.0687263\ttotal: 361ms\tremaining: 194ms\n",
      "65:\tlearn: 0.0677636\ttotal: 365ms\tremaining: 188ms\n",
      "66:\tlearn: 0.0667962\ttotal: 370ms\tremaining: 182ms\n",
      "67:\tlearn: 0.0661870\ttotal: 374ms\tremaining: 176ms\n",
      "68:\tlearn: 0.0653677\ttotal: 379ms\tremaining: 170ms\n",
      "69:\tlearn: 0.0647746\ttotal: 384ms\tremaining: 164ms\n",
      "70:\tlearn: 0.0643360\ttotal: 388ms\tremaining: 158ms\n",
      "71:\tlearn: 0.0638429\ttotal: 392ms\tremaining: 153ms\n",
      "72:\tlearn: 0.0625480\ttotal: 397ms\tremaining: 147ms\n",
      "73:\tlearn: 0.0617157\ttotal: 402ms\tremaining: 141ms\n",
      "74:\tlearn: 0.0609561\ttotal: 406ms\tremaining: 135ms\n",
      "75:\tlearn: 0.0596616\ttotal: 411ms\tremaining: 130ms\n",
      "76:\tlearn: 0.0590913\ttotal: 416ms\tremaining: 124ms\n",
      "77:\tlearn: 0.0582327\ttotal: 420ms\tremaining: 119ms\n",
      "78:\tlearn: 0.0576086\ttotal: 425ms\tremaining: 113ms\n",
      "79:\tlearn: 0.0573335\ttotal: 429ms\tremaining: 107ms\n",
      "80:\tlearn: 0.0562484\ttotal: 434ms\tremaining: 102ms\n",
      "81:\tlearn: 0.0558601\ttotal: 438ms\tremaining: 96.2ms\n",
      "82:\tlearn: 0.0555654\ttotal: 443ms\tremaining: 90.6ms\n",
      "83:\tlearn: 0.0553124\ttotal: 447ms\tremaining: 85.1ms\n",
      "84:\tlearn: 0.0550249\ttotal: 451ms\tremaining: 79.6ms\n",
      "85:\tlearn: 0.0540560\ttotal: 456ms\tremaining: 74.3ms\n",
      "86:\tlearn: 0.0537022\ttotal: 461ms\tremaining: 68.9ms\n",
      "87:\tlearn: 0.0530736\ttotal: 466ms\tremaining: 63.5ms\n",
      "88:\tlearn: 0.0522199\ttotal: 471ms\tremaining: 58.2ms\n",
      "89:\tlearn: 0.0517389\ttotal: 475ms\tremaining: 52.8ms\n",
      "90:\tlearn: 0.0515333\ttotal: 480ms\tremaining: 47.5ms\n",
      "91:\tlearn: 0.0509956\ttotal: 484ms\tremaining: 42.1ms\n",
      "92:\tlearn: 0.0507152\ttotal: 488ms\tremaining: 36.8ms\n",
      "93:\tlearn: 0.0500784\ttotal: 493ms\tremaining: 31.5ms\n",
      "94:\tlearn: 0.0498208\ttotal: 497ms\tremaining: 26.2ms\n",
      "95:\tlearn: 0.0492300\ttotal: 502ms\tremaining: 20.9ms\n",
      "96:\tlearn: 0.0489386\ttotal: 507ms\tremaining: 15.7ms\n",
      "97:\tlearn: 0.0485624\ttotal: 511ms\tremaining: 10.4ms\n",
      "98:\tlearn: 0.0483792\ttotal: 515ms\tremaining: 5.2ms\n",
      "99:\tlearn: 0.0482274\ttotal: 519ms\tremaining: 0us\n",
      "0:\tlearn: 1.3017335\ttotal: 13.1ms\tremaining: 1.3s\n",
      "1:\tlearn: 1.0961865\ttotal: 23.6ms\tremaining: 1.15s\n",
      "2:\tlearn: 0.9517011\ttotal: 33.8ms\tremaining: 1.09s\n",
      "3:\tlearn: 0.8328826\ttotal: 43.4ms\tremaining: 1.04s\n",
      "4:\tlearn: 0.7421385\ttotal: 52ms\tremaining: 988ms\n",
      "5:\tlearn: 0.6668202\ttotal: 60.5ms\tremaining: 948ms\n",
      "6:\tlearn: 0.6016396\ttotal: 68.2ms\tremaining: 906ms\n",
      "7:\tlearn: 0.5436794\ttotal: 75.9ms\tremaining: 873ms\n",
      "8:\tlearn: 0.4991022\ttotal: 82.6ms\tremaining: 835ms\n",
      "9:\tlearn: 0.4581161\ttotal: 89.5ms\tremaining: 806ms\n",
      "10:\tlearn: 0.4194359\ttotal: 95.8ms\tremaining: 775ms\n",
      "11:\tlearn: 0.3877152\ttotal: 101ms\tremaining: 742ms\n",
      "12:\tlearn: 0.3575421\ttotal: 107ms\tremaining: 715ms\n",
      "13:\tlearn: 0.3316265\ttotal: 112ms\tremaining: 690ms\n",
      "14:\tlearn: 0.3094623\ttotal: 118ms\tremaining: 668ms\n",
      "15:\tlearn: 0.2894526\ttotal: 123ms\tremaining: 647ms\n",
      "16:\tlearn: 0.2701421\ttotal: 129ms\tremaining: 628ms\n",
      "17:\tlearn: 0.2538129\ttotal: 134ms\tremaining: 609ms\n",
      "18:\tlearn: 0.2399978\ttotal: 138ms\tremaining: 590ms\n",
      "19:\tlearn: 0.2262852\ttotal: 144ms\tremaining: 574ms\n",
      "20:\tlearn: 0.2142948\ttotal: 148ms\tremaining: 558ms\n",
      "21:\tlearn: 0.2031141\ttotal: 153ms\tremaining: 544ms\n",
      "22:\tlearn: 0.1926278\ttotal: 158ms\tremaining: 529ms\n",
      "23:\tlearn: 0.1834969\ttotal: 163ms\tremaining: 516ms\n",
      "24:\tlearn: 0.1744959\ttotal: 168ms\tremaining: 504ms\n",
      "25:\tlearn: 0.1673363\ttotal: 173ms\tremaining: 492ms\n",
      "26:\tlearn: 0.1608210\ttotal: 177ms\tremaining: 480ms\n",
      "27:\tlearn: 0.1546054\ttotal: 182ms\tremaining: 468ms\n",
      "28:\tlearn: 0.1480153\ttotal: 187ms\tremaining: 458ms\n",
      "29:\tlearn: 0.1420845\ttotal: 193ms\tremaining: 449ms\n",
      "30:\tlearn: 0.1362623\ttotal: 197ms\tremaining: 439ms\n",
      "31:\tlearn: 0.1313905\ttotal: 202ms\tremaining: 430ms\n",
      "32:\tlearn: 0.1269216\ttotal: 207ms\tremaining: 421ms\n",
      "33:\tlearn: 0.1227524\ttotal: 213ms\tremaining: 413ms\n",
      "34:\tlearn: 0.1190569\ttotal: 218ms\tremaining: 404ms\n",
      "35:\tlearn: 0.1156831\ttotal: 223ms\tremaining: 396ms\n",
      "36:\tlearn: 0.1127942\ttotal: 227ms\tremaining: 387ms\n",
      "37:\tlearn: 0.1094873\ttotal: 232ms\tremaining: 378ms\n",
      "38:\tlearn: 0.1071089\ttotal: 236ms\tremaining: 370ms\n",
      "39:\tlearn: 0.1047461\ttotal: 241ms\tremaining: 361ms\n",
      "40:\tlearn: 0.1018132\ttotal: 245ms\tremaining: 353ms\n",
      "41:\tlearn: 0.0993741\ttotal: 250ms\tremaining: 345ms\n",
      "42:\tlearn: 0.0966965\ttotal: 254ms\tremaining: 337ms\n",
      "43:\tlearn: 0.0946877\ttotal: 259ms\tremaining: 330ms\n",
      "44:\tlearn: 0.0934754\ttotal: 264ms\tremaining: 323ms\n",
      "45:\tlearn: 0.0912817\ttotal: 268ms\tremaining: 315ms\n",
      "46:\tlearn: 0.0892094\ttotal: 274ms\tremaining: 308ms\n",
      "47:\tlearn: 0.0877447\ttotal: 278ms\tremaining: 301ms\n",
      "48:\tlearn: 0.0868660\ttotal: 283ms\tremaining: 294ms\n",
      "49:\tlearn: 0.0856629\ttotal: 287ms\tremaining: 287ms\n",
      "50:\tlearn: 0.0840309\ttotal: 292ms\tremaining: 280ms\n",
      "51:\tlearn: 0.0828587\ttotal: 296ms\tremaining: 273ms\n",
      "52:\tlearn: 0.0817405\ttotal: 300ms\tremaining: 266ms\n",
      "53:\tlearn: 0.0796818\ttotal: 305ms\tremaining: 260ms\n",
      "54:\tlearn: 0.0773188\ttotal: 310ms\tremaining: 253ms\n",
      "55:\tlearn: 0.0768177\ttotal: 314ms\tremaining: 247ms\n",
      "56:\tlearn: 0.0750321\ttotal: 319ms\tremaining: 240ms\n",
      "57:\tlearn: 0.0742675\ttotal: 323ms\tremaining: 234ms\n",
      "58:\tlearn: 0.0733123\ttotal: 327ms\tremaining: 227ms\n",
      "59:\tlearn: 0.0713143\ttotal: 331ms\tremaining: 221ms\n",
      "60:\tlearn: 0.0706292\ttotal: 336ms\tremaining: 215ms\n",
      "61:\tlearn: 0.0700693\ttotal: 341ms\tremaining: 209ms\n",
      "62:\tlearn: 0.0696138\ttotal: 345ms\tremaining: 202ms\n",
      "63:\tlearn: 0.0690881\ttotal: 349ms\tremaining: 196ms\n",
      "64:\tlearn: 0.0685023\ttotal: 353ms\tremaining: 190ms\n",
      "65:\tlearn: 0.0677801\ttotal: 357ms\tremaining: 184ms\n",
      "66:\tlearn: 0.0665167\ttotal: 362ms\tremaining: 178ms\n",
      "67:\tlearn: 0.0656381\ttotal: 367ms\tremaining: 173ms\n",
      "68:\tlearn: 0.0642184\ttotal: 371ms\tremaining: 167ms\n",
      "69:\tlearn: 0.0637119\ttotal: 376ms\tremaining: 161ms\n",
      "70:\tlearn: 0.0626976\ttotal: 380ms\tremaining: 155ms\n",
      "71:\tlearn: 0.0614918\ttotal: 385ms\tremaining: 150ms\n",
      "72:\tlearn: 0.0608209\ttotal: 390ms\tremaining: 144ms\n",
      "73:\tlearn: 0.0604652\ttotal: 394ms\tremaining: 138ms\n",
      "74:\tlearn: 0.0598391\ttotal: 398ms\tremaining: 133ms\n",
      "75:\tlearn: 0.0588050\ttotal: 403ms\tremaining: 127ms\n",
      "76:\tlearn: 0.0582931\ttotal: 408ms\tremaining: 122ms\n",
      "77:\tlearn: 0.0572339\ttotal: 413ms\tremaining: 116ms\n",
      "78:\tlearn: 0.0568953\ttotal: 417ms\tremaining: 111ms\n",
      "79:\tlearn: 0.0561768\ttotal: 422ms\tremaining: 105ms\n",
      "80:\tlearn: 0.0558408\ttotal: 427ms\tremaining: 100ms\n",
      "81:\tlearn: 0.0553633\ttotal: 432ms\tremaining: 94.8ms\n",
      "82:\tlearn: 0.0550752\ttotal: 436ms\tremaining: 89.3ms\n",
      "83:\tlearn: 0.0545839\ttotal: 440ms\tremaining: 83.8ms\n",
      "84:\tlearn: 0.0544000\ttotal: 444ms\tremaining: 78.4ms\n",
      "85:\tlearn: 0.0541659\ttotal: 449ms\tremaining: 73ms\n",
      "86:\tlearn: 0.0534504\ttotal: 453ms\tremaining: 67.7ms\n",
      "87:\tlearn: 0.0529305\ttotal: 458ms\tremaining: 62.4ms\n",
      "88:\tlearn: 0.0521358\ttotal: 462ms\tremaining: 57.1ms\n",
      "89:\tlearn: 0.0519983\ttotal: 466ms\tremaining: 51.8ms\n",
      "90:\tlearn: 0.0518422\ttotal: 471ms\tremaining: 46.5ms\n",
      "91:\tlearn: 0.0516932\ttotal: 475ms\tremaining: 41.3ms\n",
      "92:\tlearn: 0.0514707\ttotal: 480ms\tremaining: 36.1ms\n",
      "93:\tlearn: 0.0503761\ttotal: 485ms\tremaining: 30.9ms\n",
      "94:\tlearn: 0.0498499\ttotal: 489ms\tremaining: 25.8ms\n",
      "95:\tlearn: 0.0492255\ttotal: 494ms\tremaining: 20.6ms\n",
      "96:\tlearn: 0.0489849\ttotal: 498ms\tremaining: 15.4ms\n",
      "97:\tlearn: 0.0487908\ttotal: 502ms\tremaining: 10.3ms\n",
      "98:\tlearn: 0.0480765\ttotal: 507ms\tremaining: 5.12ms\n",
      "99:\tlearn: 0.0478708\ttotal: 511ms\tremaining: 0us\n",
      "0:\tlearn: 1.3042953\ttotal: 12.8ms\tremaining: 1.26s\n",
      "1:\tlearn: 1.1009293\ttotal: 23.6ms\tremaining: 1.16s\n",
      "2:\tlearn: 0.9593064\ttotal: 33.7ms\tremaining: 1.09s\n",
      "3:\tlearn: 0.8397413\ttotal: 42.4ms\tremaining: 1.02s\n",
      "4:\tlearn: 0.7472508\ttotal: 50.8ms\tremaining: 965ms\n",
      "5:\tlearn: 0.6693313\ttotal: 60ms\tremaining: 939ms\n",
      "6:\tlearn: 0.6029707\ttotal: 67.8ms\tremaining: 901ms\n",
      "7:\tlearn: 0.5449915\ttotal: 75.8ms\tremaining: 871ms\n",
      "8:\tlearn: 0.4992366\ttotal: 83.8ms\tremaining: 847ms\n",
      "9:\tlearn: 0.4579738\ttotal: 91.3ms\tremaining: 821ms\n",
      "10:\tlearn: 0.4206425\ttotal: 99.2ms\tremaining: 802ms\n",
      "11:\tlearn: 0.3885485\ttotal: 106ms\tremaining: 778ms\n",
      "12:\tlearn: 0.3593903\ttotal: 113ms\tremaining: 757ms\n",
      "13:\tlearn: 0.3341407\ttotal: 120ms\tremaining: 735ms\n",
      "14:\tlearn: 0.3112321\ttotal: 126ms\tremaining: 715ms\n",
      "15:\tlearn: 0.2915888\ttotal: 133ms\tremaining: 699ms\n",
      "16:\tlearn: 0.2732950\ttotal: 140ms\tremaining: 683ms\n",
      "17:\tlearn: 0.2574872\ttotal: 146ms\tremaining: 665ms\n",
      "18:\tlearn: 0.2432373\ttotal: 152ms\tremaining: 648ms\n",
      "19:\tlearn: 0.2298900\ttotal: 159ms\tremaining: 635ms\n",
      "20:\tlearn: 0.2161959\ttotal: 165ms\tremaining: 619ms\n",
      "21:\tlearn: 0.2045025\ttotal: 170ms\tremaining: 603ms\n",
      "22:\tlearn: 0.1941993\ttotal: 175ms\tremaining: 587ms\n",
      "23:\tlearn: 0.1847165\ttotal: 181ms\tremaining: 573ms\n",
      "24:\tlearn: 0.1762162\ttotal: 186ms\tremaining: 559ms\n",
      "25:\tlearn: 0.1691035\ttotal: 192ms\tremaining: 545ms\n",
      "26:\tlearn: 0.1620943\ttotal: 197ms\tremaining: 531ms\n",
      "27:\tlearn: 0.1557906\ttotal: 202ms\tremaining: 518ms\n",
      "28:\tlearn: 0.1493144\ttotal: 207ms\tremaining: 508ms\n",
      "29:\tlearn: 0.1441082\ttotal: 212ms\tremaining: 494ms\n",
      "30:\tlearn: 0.1388749\ttotal: 217ms\tremaining: 482ms\n",
      "31:\tlearn: 0.1337878\ttotal: 222ms\tremaining: 472ms\n",
      "32:\tlearn: 0.1293109\ttotal: 226ms\tremaining: 460ms\n",
      "33:\tlearn: 0.1246831\ttotal: 231ms\tremaining: 448ms\n",
      "34:\tlearn: 0.1201904\ttotal: 236ms\tremaining: 437ms\n",
      "35:\tlearn: 0.1162081\ttotal: 240ms\tremaining: 427ms\n",
      "36:\tlearn: 0.1125625\ttotal: 245ms\tremaining: 418ms\n",
      "37:\tlearn: 0.1102490\ttotal: 250ms\tremaining: 407ms\n",
      "38:\tlearn: 0.1075696\ttotal: 254ms\tremaining: 398ms\n",
      "39:\tlearn: 0.1039248\ttotal: 259ms\tremaining: 389ms\n",
      "40:\tlearn: 0.1009414\ttotal: 264ms\tremaining: 380ms\n",
      "41:\tlearn: 0.0982821\ttotal: 269ms\tremaining: 371ms\n",
      "42:\tlearn: 0.0965895\ttotal: 273ms\tremaining: 362ms\n",
      "43:\tlearn: 0.0942084\ttotal: 278ms\tremaining: 353ms\n",
      "44:\tlearn: 0.0927438\ttotal: 282ms\tremaining: 345ms\n",
      "45:\tlearn: 0.0909280\ttotal: 287ms\tremaining: 336ms\n",
      "46:\tlearn: 0.0889423\ttotal: 291ms\tremaining: 329ms\n",
      "47:\tlearn: 0.0875365\ttotal: 296ms\tremaining: 321ms\n",
      "48:\tlearn: 0.0857594\ttotal: 300ms\tremaining: 313ms\n",
      "49:\tlearn: 0.0834438\ttotal: 305ms\tremaining: 305ms\n",
      "50:\tlearn: 0.0822498\ttotal: 309ms\tremaining: 297ms\n",
      "51:\tlearn: 0.0813812\ttotal: 314ms\tremaining: 290ms\n",
      "52:\tlearn: 0.0803896\ttotal: 319ms\tremaining: 282ms\n",
      "53:\tlearn: 0.0795679\ttotal: 323ms\tremaining: 275ms\n",
      "54:\tlearn: 0.0785627\ttotal: 328ms\tremaining: 268ms\n",
      "55:\tlearn: 0.0776362\ttotal: 332ms\tremaining: 261ms\n",
      "56:\tlearn: 0.0763618\ttotal: 337ms\tremaining: 254ms\n",
      "57:\tlearn: 0.0758618\ttotal: 341ms\tremaining: 247ms\n",
      "58:\tlearn: 0.0744004\ttotal: 346ms\tremaining: 240ms\n",
      "59:\tlearn: 0.0726560\ttotal: 351ms\tremaining: 234ms\n",
      "60:\tlearn: 0.0710921\ttotal: 356ms\tremaining: 227ms\n",
      "61:\tlearn: 0.0702467\ttotal: 360ms\tremaining: 221ms\n",
      "62:\tlearn: 0.0691036\ttotal: 366ms\tremaining: 215ms\n",
      "63:\tlearn: 0.0685562\ttotal: 370ms\tremaining: 208ms\n",
      "64:\tlearn: 0.0680877\ttotal: 375ms\tremaining: 202ms\n",
      "65:\tlearn: 0.0674205\ttotal: 379ms\tremaining: 195ms\n",
      "66:\tlearn: 0.0669605\ttotal: 385ms\tremaining: 189ms\n",
      "67:\tlearn: 0.0662319\ttotal: 389ms\tremaining: 183ms\n",
      "68:\tlearn: 0.0653795\ttotal: 393ms\tremaining: 177ms\n",
      "69:\tlearn: 0.0646932\ttotal: 397ms\tremaining: 170ms\n",
      "70:\tlearn: 0.0643319\ttotal: 402ms\tremaining: 164ms\n",
      "71:\tlearn: 0.0634487\ttotal: 406ms\tremaining: 158ms\n",
      "72:\tlearn: 0.0621734\ttotal: 411ms\tremaining: 152ms\n",
      "73:\tlearn: 0.0619488\ttotal: 415ms\tremaining: 146ms\n",
      "74:\tlearn: 0.0615775\ttotal: 419ms\tremaining: 140ms\n",
      "75:\tlearn: 0.0610949\ttotal: 424ms\tremaining: 134ms\n",
      "76:\tlearn: 0.0604131\ttotal: 429ms\tremaining: 128ms\n",
      "77:\tlearn: 0.0597618\ttotal: 434ms\tremaining: 122ms\n",
      "78:\tlearn: 0.0594349\ttotal: 438ms\tremaining: 116ms\n",
      "79:\tlearn: 0.0589255\ttotal: 443ms\tremaining: 111ms\n",
      "80:\tlearn: 0.0580754\ttotal: 447ms\tremaining: 105ms\n",
      "81:\tlearn: 0.0575266\ttotal: 452ms\tremaining: 99.2ms\n",
      "82:\tlearn: 0.0567481\ttotal: 456ms\tremaining: 93.5ms\n",
      "83:\tlearn: 0.0564910\ttotal: 461ms\tremaining: 87.7ms\n",
      "84:\tlearn: 0.0561469\ttotal: 465ms\tremaining: 82.1ms\n",
      "85:\tlearn: 0.0554627\ttotal: 470ms\tremaining: 76.6ms\n",
      "86:\tlearn: 0.0552025\ttotal: 475ms\tremaining: 70.9ms\n",
      "87:\tlearn: 0.0548129\ttotal: 479ms\tremaining: 65.3ms\n",
      "88:\tlearn: 0.0546404\ttotal: 483ms\tremaining: 59.7ms\n",
      "89:\tlearn: 0.0544244\ttotal: 488ms\tremaining: 54.2ms\n",
      "90:\tlearn: 0.0534345\ttotal: 493ms\tremaining: 48.7ms\n",
      "91:\tlearn: 0.0529918\ttotal: 497ms\tremaining: 43.2ms\n",
      "92:\tlearn: 0.0527978\ttotal: 501ms\tremaining: 37.7ms\n",
      "93:\tlearn: 0.0526169\ttotal: 505ms\tremaining: 32.3ms\n",
      "94:\tlearn: 0.0524019\ttotal: 510ms\tremaining: 26.8ms\n",
      "95:\tlearn: 0.0521472\ttotal: 514ms\tremaining: 21.4ms\n",
      "96:\tlearn: 0.0514899\ttotal: 519ms\tremaining: 16ms\n",
      "97:\tlearn: 0.0513302\ttotal: 523ms\tremaining: 10.7ms\n",
      "98:\tlearn: 0.0504422\ttotal: 528ms\tremaining: 5.33ms\n",
      "99:\tlearn: 0.0502212\ttotal: 532ms\tremaining: 0us\n",
      "0:\tlearn: 1.3062043\ttotal: 13.5ms\tremaining: 1.34s\n",
      "1:\tlearn: 1.1062557\ttotal: 24.9ms\tremaining: 1.22s\n",
      "2:\tlearn: 0.9617928\ttotal: 35.2ms\tremaining: 1.14s\n",
      "3:\tlearn: 0.8436505\ttotal: 44.8ms\tremaining: 1.07s\n",
      "4:\tlearn: 0.7516927\ttotal: 53.3ms\tremaining: 1.01s\n",
      "5:\tlearn: 0.6755258\ttotal: 61.7ms\tremaining: 966ms\n",
      "6:\tlearn: 0.6101596\ttotal: 70ms\tremaining: 930ms\n",
      "7:\tlearn: 0.5547323\ttotal: 78.4ms\tremaining: 901ms\n",
      "8:\tlearn: 0.5095086\ttotal: 85.5ms\tremaining: 864ms\n",
      "9:\tlearn: 0.4652967\ttotal: 91.9ms\tremaining: 827ms\n",
      "10:\tlearn: 0.4277575\ttotal: 98ms\tremaining: 793ms\n",
      "11:\tlearn: 0.3963399\ttotal: 104ms\tremaining: 762ms\n",
      "12:\tlearn: 0.3668452\ttotal: 110ms\tremaining: 737ms\n",
      "13:\tlearn: 0.3406807\ttotal: 116ms\tremaining: 714ms\n",
      "14:\tlearn: 0.3180595\ttotal: 122ms\tremaining: 689ms\n",
      "15:\tlearn: 0.2984138\ttotal: 127ms\tremaining: 668ms\n",
      "16:\tlearn: 0.2786142\ttotal: 134ms\tremaining: 654ms\n",
      "17:\tlearn: 0.2625713\ttotal: 139ms\tremaining: 635ms\n",
      "18:\tlearn: 0.2485963\ttotal: 145ms\tremaining: 619ms\n",
      "19:\tlearn: 0.2357249\ttotal: 151ms\tremaining: 603ms\n",
      "20:\tlearn: 0.2231021\ttotal: 156ms\tremaining: 587ms\n",
      "21:\tlearn: 0.2113605\ttotal: 161ms\tremaining: 571ms\n",
      "22:\tlearn: 0.2007333\ttotal: 166ms\tremaining: 555ms\n",
      "23:\tlearn: 0.1907009\ttotal: 171ms\tremaining: 541ms\n",
      "24:\tlearn: 0.1815160\ttotal: 176ms\tremaining: 529ms\n",
      "25:\tlearn: 0.1736591\ttotal: 181ms\tremaining: 515ms\n",
      "26:\tlearn: 0.1667577\ttotal: 186ms\tremaining: 503ms\n",
      "27:\tlearn: 0.1596993\ttotal: 191ms\tremaining: 491ms\n",
      "28:\tlearn: 0.1533693\ttotal: 196ms\tremaining: 481ms\n",
      "29:\tlearn: 0.1481129\ttotal: 201ms\tremaining: 469ms\n",
      "30:\tlearn: 0.1430700\ttotal: 205ms\tremaining: 457ms\n",
      "31:\tlearn: 0.1383590\ttotal: 211ms\tremaining: 447ms\n",
      "32:\tlearn: 0.1339277\ttotal: 216ms\tremaining: 438ms\n",
      "33:\tlearn: 0.1289948\ttotal: 220ms\tremaining: 428ms\n",
      "34:\tlearn: 0.1255579\ttotal: 225ms\tremaining: 418ms\n",
      "35:\tlearn: 0.1218215\ttotal: 230ms\tremaining: 409ms\n",
      "36:\tlearn: 0.1185890\ttotal: 235ms\tremaining: 399ms\n",
      "37:\tlearn: 0.1161212\ttotal: 240ms\tremaining: 391ms\n",
      "38:\tlearn: 0.1135874\ttotal: 244ms\tremaining: 382ms\n",
      "39:\tlearn: 0.1095170\ttotal: 249ms\tremaining: 374ms\n",
      "40:\tlearn: 0.1077087\ttotal: 254ms\tremaining: 365ms\n",
      "41:\tlearn: 0.1051579\ttotal: 258ms\tremaining: 357ms\n",
      "42:\tlearn: 0.1028508\ttotal: 262ms\tremaining: 348ms\n",
      "43:\tlearn: 0.1013901\ttotal: 267ms\tremaining: 339ms\n",
      "44:\tlearn: 0.0996929\ttotal: 271ms\tremaining: 331ms\n",
      "45:\tlearn: 0.0976946\ttotal: 276ms\tremaining: 324ms\n",
      "46:\tlearn: 0.0959312\ttotal: 281ms\tremaining: 316ms\n",
      "47:\tlearn: 0.0949906\ttotal: 285ms\tremaining: 308ms\n",
      "48:\tlearn: 0.0932577\ttotal: 290ms\tremaining: 302ms\n",
      "49:\tlearn: 0.0920243\ttotal: 294ms\tremaining: 294ms\n",
      "50:\tlearn: 0.0909184\ttotal: 298ms\tremaining: 287ms\n",
      "51:\tlearn: 0.0884593\ttotal: 303ms\tremaining: 280ms\n",
      "52:\tlearn: 0.0869038\ttotal: 307ms\tremaining: 273ms\n",
      "53:\tlearn: 0.0849540\ttotal: 313ms\tremaining: 266ms\n",
      "54:\tlearn: 0.0835793\ttotal: 317ms\tremaining: 259ms\n",
      "55:\tlearn: 0.0826265\ttotal: 321ms\tremaining: 253ms\n",
      "56:\tlearn: 0.0817512\ttotal: 326ms\tremaining: 246ms\n",
      "57:\tlearn: 0.0809370\ttotal: 331ms\tremaining: 239ms\n",
      "58:\tlearn: 0.0792970\ttotal: 335ms\tremaining: 233ms\n",
      "59:\tlearn: 0.0771548\ttotal: 341ms\tremaining: 227ms\n",
      "60:\tlearn: 0.0766221\ttotal: 345ms\tremaining: 221ms\n",
      "61:\tlearn: 0.0756019\ttotal: 349ms\tremaining: 214ms\n",
      "62:\tlearn: 0.0745939\ttotal: 354ms\tremaining: 208ms\n",
      "63:\tlearn: 0.0734200\ttotal: 359ms\tremaining: 202ms\n",
      "64:\tlearn: 0.0726498\ttotal: 364ms\tremaining: 196ms\n",
      "65:\tlearn: 0.0722699\ttotal: 368ms\tremaining: 190ms\n",
      "66:\tlearn: 0.0717601\ttotal: 372ms\tremaining: 183ms\n",
      "67:\tlearn: 0.0711362\ttotal: 377ms\tremaining: 177ms\n",
      "68:\tlearn: 0.0699097\ttotal: 382ms\tremaining: 172ms\n",
      "69:\tlearn: 0.0684489\ttotal: 387ms\tremaining: 166ms\n",
      "70:\tlearn: 0.0670426\ttotal: 391ms\tremaining: 160ms\n",
      "71:\tlearn: 0.0663753\ttotal: 396ms\tremaining: 154ms\n",
      "72:\tlearn: 0.0652326\ttotal: 401ms\tremaining: 148ms\n",
      "73:\tlearn: 0.0648860\ttotal: 405ms\tremaining: 142ms\n",
      "74:\tlearn: 0.0637882\ttotal: 409ms\tremaining: 136ms\n",
      "75:\tlearn: 0.0628768\ttotal: 414ms\tremaining: 131ms\n",
      "76:\tlearn: 0.0624872\ttotal: 418ms\tremaining: 125ms\n",
      "77:\tlearn: 0.0618093\ttotal: 422ms\tremaining: 119ms\n",
      "78:\tlearn: 0.0613905\ttotal: 427ms\tremaining: 113ms\n",
      "79:\tlearn: 0.0607034\ttotal: 431ms\tremaining: 108ms\n",
      "80:\tlearn: 0.0598583\ttotal: 436ms\tremaining: 102ms\n",
      "81:\tlearn: 0.0594122\ttotal: 441ms\tremaining: 96.7ms\n",
      "82:\tlearn: 0.0591141\ttotal: 445ms\tremaining: 91.1ms\n",
      "83:\tlearn: 0.0581445\ttotal: 450ms\tremaining: 85.7ms\n",
      "84:\tlearn: 0.0577985\ttotal: 454ms\tremaining: 80.2ms\n",
      "85:\tlearn: 0.0573816\ttotal: 459ms\tremaining: 74.8ms\n",
      "86:\tlearn: 0.0570244\ttotal: 465ms\tremaining: 69.4ms\n",
      "87:\tlearn: 0.0565181\ttotal: 469ms\tremaining: 64ms\n",
      "88:\tlearn: 0.0560368\ttotal: 474ms\tremaining: 58.6ms\n",
      "89:\tlearn: 0.0556176\ttotal: 478ms\tremaining: 53.1ms\n",
      "90:\tlearn: 0.0551205\ttotal: 483ms\tremaining: 47.7ms\n",
      "91:\tlearn: 0.0548802\ttotal: 487ms\tremaining: 42.3ms\n",
      "92:\tlearn: 0.0544960\ttotal: 491ms\tremaining: 37ms\n",
      "93:\tlearn: 0.0539508\ttotal: 496ms\tremaining: 31.6ms\n",
      "94:\tlearn: 0.0533720\ttotal: 500ms\tremaining: 26.3ms\n",
      "95:\tlearn: 0.0527881\ttotal: 505ms\tremaining: 21ms\n",
      "96:\tlearn: 0.0525461\ttotal: 509ms\tremaining: 15.7ms\n",
      "97:\tlearn: 0.0522827\ttotal: 514ms\tremaining: 10.5ms\n",
      "98:\tlearn: 0.0514740\ttotal: 519ms\tremaining: 5.24ms\n",
      "99:\tlearn: 0.0512778\ttotal: 523ms\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6814.0    39.0    22.0    5.0  0.0\n",
      "1.0    63.0  4755.0    15.0    2.0  0.0\n",
      "2.0     5.0    10.0  1256.0   20.0  2.0\n",
      "3.0     3.0     0.0     7.0  339.0  0.0\n",
      "4.0     0.0     0.0     2.0    2.0  6.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9852622129123962\n",
      "Precision total:  0.9230706569015446\n",
      "Recall total:  0.9033184080835671\n",
      "F1 total:  0.9113905731337114\n",
      "BACC total:  0.9033184080835671\n",
      "MCC total:  0.9752273088859951\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baggin LGBM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6815.0    32.0    24.0    9.0  0.0\n",
      "1.0    21.0  4803.0     9.0    2.0  0.0\n",
      "2.0     7.0     4.0  1276.0    4.0  2.0\n",
      "3.0     3.0     2.0     4.0  339.0  1.0\n",
      "4.0     0.0     0.0     2.0    1.0  7.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9904989900501234\n",
      "Precision total:  0.922578794659135\n",
      "Recall total:  0.9284265809045203\n",
      "F1 total:  0.9254734737732246\n",
      "BACC total:  0.9284265809045203\n",
      "MCC total:  0.9840461098486635\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Create a DMatrix for XGBoost\n",
    "# dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "# dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# # Set XGBoost parameters\n",
    "# params = {\n",
    "#     'objective': 'multi:softmax',  # for multi-class classification\n",
    "#     'num_class': 5,  # specify the number of classes\n",
    "#     'max_depth': 3,\n",
    "#     'learning_rate': 0.1,\n",
    "#     'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "# }\n",
    "\n",
    "# # Train the XGBoost model\n",
    "# num_round = 100\n",
    "# xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# base_classifier = xgb\n",
    "\n",
    "# # Define the BaggingClassifier\n",
    "# bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# # Train the BaggingClassifier\n",
    "# bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "# name = 'bag_xgb'\n",
    "\n",
    "# pred_label = y_pred\n",
    "\n",
    "\n",
    "# metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "# Acc = metrics[0]\n",
    "# Precision = metrics[1]\n",
    "# Recall = metrics[2]\n",
    "# F1 = metrics[3]\n",
    "# BACC = metrics[4]\n",
    "# MCC = metrics[5]    \n",
    "\n",
    "\n",
    "# globals()[f\"{name}_acc_01\"] = Acc\n",
    "# globals()[f\"{name}_pre_01\"] = Precision\n",
    "# globals()[f\"{name}_rec_01\"] = Recall\n",
    "# globals()[f\"{name}_f1_01\"] = F1\n",
    "# globals()[f\"{name}_bacc_01\"] = BACC\n",
    "# globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6758.0    53.0    50.0   19.0  0.0\n",
      "1.0   171.0  4584.0    39.0   41.0  0.0\n",
      "2.0     0.0     4.0  1245.0   44.0  0.0\n",
      "3.0     0.0     2.0    13.0  334.0  0.0\n",
      "4.0     0.0     0.0     3.0    7.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9666342485224807\n",
      "Precision total:  0.7270795665638226\n",
      "Recall total:  0.7700502791854386\n",
      "F1 total:  0.7458991384972744\n",
      "BACC total:  0.7700502791854386\n",
      "MCC total:  0.9443415027341262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging with many models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### do bootstrapping "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train_01.assign(label = y_train_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rf</th>\n",
       "      <th>ada</th>\n",
       "      <th>xgb</th>\n",
       "      <th>cat</th>\n",
       "      <th>dnn</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.806821</td>\n",
       "      <td>0.254891</td>\n",
       "      <td>0.972982</td>\n",
       "      <td>0.985371</td>\n",
       "      <td>0.577608</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.966415</td>\n",
       "      <td>0.255779</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.996741</td>\n",
       "      <td>0.988606</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.847230</td>\n",
       "      <td>0.242380</td>\n",
       "      <td>0.993120</td>\n",
       "      <td>0.988092</td>\n",
       "      <td>0.940237</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.946073</td>\n",
       "      <td>0.257206</td>\n",
       "      <td>0.992883</td>\n",
       "      <td>0.996443</td>\n",
       "      <td>0.976056</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.683130</td>\n",
       "      <td>0.295701</td>\n",
       "      <td>0.946512</td>\n",
       "      <td>0.975953</td>\n",
       "      <td>0.209521</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31184</th>\n",
       "      <td>0.142267</td>\n",
       "      <td>0.231585</td>\n",
       "      <td>0.910927</td>\n",
       "      <td>0.597961</td>\n",
       "      <td>0.077612</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31185</th>\n",
       "      <td>0.994399</td>\n",
       "      <td>0.328195</td>\n",
       "      <td>0.997686</td>\n",
       "      <td>0.997247</td>\n",
       "      <td>0.979955</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31186</th>\n",
       "      <td>0.966415</td>\n",
       "      <td>0.255779</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.996609</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31187</th>\n",
       "      <td>0.868457</td>\n",
       "      <td>0.248905</td>\n",
       "      <td>0.995268</td>\n",
       "      <td>0.991756</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31188</th>\n",
       "      <td>0.983763</td>\n",
       "      <td>0.296489</td>\n",
       "      <td>0.997746</td>\n",
       "      <td>0.994873</td>\n",
       "      <td>0.999144</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31189 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             rf       ada       xgb       cat       dnn  label\n",
       "0      0.806821  0.254891  0.972982  0.985371  0.577608    0.0\n",
       "1      0.966415  0.255779  0.994128  0.996741  0.988606    0.0\n",
       "2      0.847230  0.242380  0.993120  0.988092  0.940237    1.0\n",
       "3      0.946073  0.257206  0.992883  0.996443  0.976056    0.0\n",
       "4      0.683130  0.295701  0.946512  0.975953  0.209521    0.0\n",
       "...         ...       ...       ...       ...       ...    ...\n",
       "31184  0.142267  0.231585  0.910927  0.597961  0.077612    3.0\n",
       "31185  0.994399  0.328195  0.997686  0.997247  0.979955    1.0\n",
       "31186  0.966415  0.255779  0.994128  0.996609  0.980899    0.0\n",
       "31187  0.868457  0.248905  0.995268  0.991756  0.999921    0.0\n",
       "31188  0.983763  0.296489  0.997746  0.994873  0.999144    1.0\n",
       "\n",
       "[31189 rows x 6 columns]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boot_df[5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.A base model (weak model) is created on each of these subsets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_comb_pred = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVM\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_svm_01)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ADA\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train_01, y_train_01)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_01 = ada.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_ada_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.2987524\ttest: 1.3004348\tbest: 1.3004348 (0)\ttotal: 13.4ms\tremaining: 1.33s\n",
      "10:\tlearn: 0.4186650\ttest: 0.4228686\tbest: 0.4228686 (10)\ttotal: 97.3ms\tremaining: 787ms\n",
      "20:\tlearn: 0.2125330\ttest: 0.2191971\tbest: 0.2191971 (20)\ttotal: 154ms\tremaining: 579ms\n",
      "30:\tlearn: 0.1326798\ttest: 0.1404523\tbest: 0.1404523 (30)\ttotal: 202ms\tremaining: 450ms\n",
      "40:\tlearn: 0.0980108\ttest: 0.1060605\tbest: 0.1060605 (40)\ttotal: 251ms\tremaining: 362ms\n",
      "50:\tlearn: 0.0789497\ttest: 0.0875761\tbest: 0.0875761 (50)\ttotal: 298ms\tremaining: 286ms\n",
      "60:\tlearn: 0.0682801\ttest: 0.0774007\tbest: 0.0774007 (60)\ttotal: 343ms\tremaining: 219ms\n",
      "70:\tlearn: 0.0589124\ttest: 0.0683745\tbest: 0.0683745 (70)\ttotal: 389ms\tremaining: 159ms\n",
      "80:\tlearn: 0.0536712\ttest: 0.0636196\tbest: 0.0636196 (80)\ttotal: 434ms\tremaining: 102ms\n",
      "90:\tlearn: 0.0484589\ttest: 0.0587808\tbest: 0.0587808 (90)\ttotal: 479ms\tremaining: 47.4ms\n",
      "99:\tlearn: 0.0447801\ttest: 0.0555799\tbest: 0.0555799 (99)\ttotal: 521ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.055579906\n",
      "bestIteration = 99\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Catboost\n",
    "import catboost\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_01.fit(X_train_boot, y_train_boot, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#LGBM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_lgbm_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "#KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_01.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_01 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_rf_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195/195 [==============================] - 1s 2ms/step - loss: 1.5372 - accuracy: 0.3016 - val_loss: 1.2627 - val_accuracy: 0.5188\n",
      "Epoch 2/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 1.0800 - accuracy: 0.5124 - val_loss: 0.9463 - val_accuracy: 0.5247\n",
      "Epoch 3/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8730 - accuracy: 0.5312 - val_loss: 0.7689 - val_accuracy: 0.5604\n",
      "Epoch 4/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.7734 - accuracy: 0.5952 - val_loss: 0.7199 - val_accuracy: 0.6199\n",
      "Epoch 5/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.7354 - accuracy: 0.6119 - val_loss: 0.6784 - val_accuracy: 0.7486\n",
      "Epoch 6/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.6942 - accuracy: 0.7011 - val_loss: 0.6225 - val_accuracy: 0.8315\n",
      "Epoch 7/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.6433 - accuracy: 0.7827 - val_loss: 0.5754 - val_accuracy: 0.8254\n",
      "Epoch 8/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.6106 - accuracy: 0.7957 - val_loss: 0.5453 - val_accuracy: 0.8492\n",
      "Epoch 9/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5917 - accuracy: 0.7996 - val_loss: 0.5306 - val_accuracy: 0.8549\n",
      "Epoch 10/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5807 - accuracy: 0.8025 - val_loss: 0.5201 - val_accuracy: 0.8567\n",
      "Epoch 11/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5704 - accuracy: 0.8055 - val_loss: 0.5116 - val_accuracy: 0.8426\n",
      "Epoch 12/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5603 - accuracy: 0.8058 - val_loss: 0.5034 - val_accuracy: 0.8445\n",
      "Epoch 13/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5552 - accuracy: 0.8068 - val_loss: 0.4979 - val_accuracy: 0.8450\n",
      "Epoch 14/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5514 - accuracy: 0.8053 - val_loss: 0.4942 - val_accuracy: 0.8623\n",
      "Epoch 15/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5422 - accuracy: 0.8122 - val_loss: 0.4894 - val_accuracy: 0.8503\n",
      "Epoch 16/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5427 - accuracy: 0.8112 - val_loss: 0.4852 - val_accuracy: 0.8447\n",
      "Epoch 17/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5371 - accuracy: 0.8142 - val_loss: 0.4808 - val_accuracy: 0.8584\n",
      "Epoch 18/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5397 - accuracy: 0.8154 - val_loss: 0.4782 - val_accuracy: 0.8556\n",
      "Epoch 19/100\n",
      "195/195 [==============================] - 0s 3ms/step - loss: 0.5323 - accuracy: 0.8158 - val_loss: 0.4772 - val_accuracy: 0.8415\n",
      "Epoch 20/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5290 - accuracy: 0.8204 - val_loss: 0.4753 - val_accuracy: 0.8431\n",
      "Epoch 21/100\n",
      "195/195 [==============================] - 1s 3ms/step - loss: 0.5237 - accuracy: 0.8228 - val_loss: 0.4726 - val_accuracy: 0.8455\n",
      "Epoch 22/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5229 - accuracy: 0.8235 - val_loss: 0.4677 - val_accuracy: 0.8700\n",
      "Epoch 23/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5181 - accuracy: 0.8279 - val_loss: 0.4657 - val_accuracy: 0.8525\n",
      "Epoch 24/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5150 - accuracy: 0.8285 - val_loss: 0.4654 - val_accuracy: 0.8580\n",
      "Epoch 25/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5157 - accuracy: 0.8288 - val_loss: 0.4629 - val_accuracy: 0.8500\n",
      "Epoch 26/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5127 - accuracy: 0.8271 - val_loss: 0.4597 - val_accuracy: 0.8625\n",
      "Epoch 27/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5079 - accuracy: 0.8299 - val_loss: 0.4580 - val_accuracy: 0.8621\n",
      "Epoch 28/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5018 - accuracy: 0.8351 - val_loss: 0.4578 - val_accuracy: 0.8695\n",
      "Epoch 29/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5017 - accuracy: 0.8350 - val_loss: 0.4533 - val_accuracy: 0.8679\n",
      "Epoch 30/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5052 - accuracy: 0.8348 - val_loss: 0.4531 - val_accuracy: 0.8650\n",
      "Epoch 31/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.5027 - accuracy: 0.8334 - val_loss: 0.4527 - val_accuracy: 0.8596\n",
      "Epoch 32/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.4976 - accuracy: 0.8368 - val_loss: 0.4515 - val_accuracy: 0.8693\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.02\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "# dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "from keras.callbacks import EarlyStopping\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_01.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_01.predict(X_test_01)\n",
    "preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "#LogReg\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg_01 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_01.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_01.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_logreg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. The models run in parallel and are independent of each other."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0          1.0      0.0      1.0        1      1.0      1.0      1.0        1   \n",
      "1          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "2          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "3          2.0      2.0      2.0        2      2.0      2.0      2.0        2   \n",
      "4          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "13362      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "13363      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "13364      2.0      0.0      0.0        2      2.0      0.0      2.0        2   \n",
      "13365      0.0      0.0      0.0        0      1.0      0.0      0.0        0   \n",
      "13366      2.0      2.0      2.0        2      2.0      2.0      2.0        2   \n",
      "\n",
      "       model_8  model_9  \n",
      "0          1.0      1.0  \n",
      "1          0.0      0.0  \n",
      "2          0.0      0.0  \n",
      "3          2.0      2.0  \n",
      "4          0.0      0.0  \n",
      "...        ...      ...  \n",
      "13362      0.0      0.0  \n",
      "13363      0.0      0.0  \n",
      "13364      2.0      0.0  \n",
      "13365      0.0      0.0  \n",
      "13366      2.0      2.0  \n",
      "\n",
      "[13367 rows x 10 columns]\n"
     ]
    }
   ],
   "source": [
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0          1.0      0.0      1.0        1      1.0      1.0      1.0        1   \n",
      "1          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "2          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "3          2.0      2.0      2.0        2      2.0      2.0      2.0        2   \n",
      "4          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "13362      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "13363      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "13364      2.0      0.0      0.0        2      2.0      0.0      2.0        2   \n",
      "13365      0.0      0.0      0.0        0      1.0      0.0      0.0        0   \n",
      "13366      2.0      2.0      2.0        2      2.0      2.0      2.0        2   \n",
      "\n",
      "       model_8  model_9  ensemble  \n",
      "0          1.0      1.0         1  \n",
      "1          0.0      0.0         0  \n",
      "2          0.0      0.0         0  \n",
      "3          2.0      2.0         2  \n",
      "4          0.0      0.0         0  \n",
      "...        ...      ...       ...  \n",
      "13362      0.0      0.0         0  \n",
      "13363      0.0      0.0         0  \n",
      "13364      2.0      0.0         2  \n",
      "13365      0.0      0.0         0  \n",
      "13366      2.0      2.0         2  \n",
      "\n",
      "[13367 rows x 11 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        1\n",
       "1        0\n",
       "2        0\n",
       "3        2\n",
       "4        0\n",
       "        ..\n",
       "13362    0\n",
       "13363    0\n",
       "13364    2\n",
       "13365    0\n",
       "13366    2\n",
       "Name: ensemble, Length: 13367, dtype: int64"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Voting start\n",
    "from scipy.stats import mode\n",
    "# bag_comb_pred_df = pd.DataFrame(bag_comb_pred)\n",
    "# Extract predictions columns\n",
    "\n",
    "# predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "    # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# predictions = bag_comb_pred_df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1       2      3    4\n",
      "0  6785.0    27.0    60.0    8.0  0.0\n",
      "1   483.0  4303.0    44.0    5.0  0.0\n",
      "2     9.0     1.0  1245.0   38.0  0.0\n",
      "3     0.0     0.0    23.0  326.0  0.0\n",
      "4     3.0     0.0     1.0    5.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9471085509089548\n",
      "Precision total:  0.9371434209903302\n",
      "Recall total:  0.7746270576092131\n",
      "F1 total:  0.7809937046410604\n",
      "BACC total:  0.7746270576092131\n",
      "MCC total:  0.9124622627677437\n"
     ]
    }
   ],
   "source": [
    "name='bag_comb'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_18 (Dense)             (None, 3)                 18        \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_16 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_17 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_18 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 86\n",
      "Trainable params: 86\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_dnn = time.time()\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train_01.shape[1]\n",
    "\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(3*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "# dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn_01.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "195/195 [==============================] - ETA: 0s - loss: 1.4826 - accuracy: 0.3294"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195/195 [==============================] - 1s 3ms/step - loss: 1.4826 - accuracy: 0.3294 - val_loss: 1.2129 - val_accuracy: 0.3581\n",
      "Epoch 2/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 1.1699 - accuracy: 0.4728 - val_loss: 1.0232 - val_accuracy: 0.5289\n",
      "Epoch 3/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 1.0992 - accuracy: 0.5145 - val_loss: 1.0044 - val_accuracy: 0.5289\n",
      "Epoch 4/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 1.0642 - accuracy: 0.5174 - val_loss: 0.9771 - val_accuracy: 0.5289\n",
      "Epoch 5/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.9976 - accuracy: 0.5189 - val_loss: 0.8779 - val_accuracy: 0.5290\n",
      "Epoch 6/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.9408 - accuracy: 0.5217 - val_loss: 0.8357 - val_accuracy: 0.5436\n",
      "Epoch 7/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.9063 - accuracy: 0.5281 - val_loss: 0.8055 - val_accuracy: 0.5561\n",
      "Epoch 8/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8762 - accuracy: 0.5503 - val_loss: 0.7814 - val_accuracy: 0.5850\n",
      "Epoch 9/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8517 - accuracy: 0.5660 - val_loss: 0.7784 - val_accuracy: 0.5854\n",
      "Epoch 10/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8453 - accuracy: 0.5686 - val_loss: 0.7713 - val_accuracy: 0.6045\n",
      "Epoch 11/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8379 - accuracy: 0.5726 - val_loss: 0.7708 - val_accuracy: 0.6032\n",
      "Epoch 12/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8363 - accuracy: 0.5725 - val_loss: 0.7686 - val_accuracy: 0.6032\n",
      "Epoch 13/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8288 - accuracy: 0.5755 - val_loss: 0.7650 - val_accuracy: 0.6034\n",
      "Epoch 14/100\n",
      "195/195 [==============================] - 0s 3ms/step - loss: 0.8259 - accuracy: 0.5750 - val_loss: 0.7667 - val_accuracy: 0.6032\n",
      "Epoch 15/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8237 - accuracy: 0.5756 - val_loss: 0.7665 - val_accuracy: 0.6034\n",
      "Epoch 16/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8191 - accuracy: 0.5785 - val_loss: 0.7639 - val_accuracy: 0.6034\n",
      "Epoch 17/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8196 - accuracy: 0.5789 - val_loss: 0.7621 - val_accuracy: 0.6032\n",
      "Epoch 18/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8138 - accuracy: 0.5799 - val_loss: 0.7628 - val_accuracy: 0.6044\n",
      "Epoch 19/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8192 - accuracy: 0.5807 - val_loss: 0.7622 - val_accuracy: 0.6032\n",
      "Epoch 20/100\n",
      "195/195 [==============================] - 0s 2ms/step - loss: 0.8109 - accuracy: 0.5816 - val_loss: 0.7620 - val_accuracy: 0.6036\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "try:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size)\n",
    "    dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # model.fit(x_train, Y_train, epochs=100, batch_size=128, validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(dnn_01, 'dnn_level_01.joblib')\n",
    "    # dnn_01.save(\"dnn_level_01.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "except: \n",
    "    None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dnn_01 = load_model(\"dnn_level_01.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DNN\n",
    "try:\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn_01.predict(X_test_01)\n",
    "    preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "except:\n",
    "        with open(output_file_name, \"a\") as f: print('error', file = f)\n",
    "        preds_dnn_01 = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0  2.0  3.0  4.0\n",
      "0.0  6734.0   146.0  0.0  0.0  0.0\n",
      "1.0  4646.0   189.0  0.0  0.0  0.0\n",
      "2.0    21.0  1272.0  0.0  0.0  0.0\n",
      "3.0     0.0   349.0  0.0  0.0  0.0\n",
      "4.0     0.0    10.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.598937682352061\n",
      "Precision total:  0.24752978513869187\n",
      "Recall total:  0.39250755409269955\n",
      "F1 total:  0.30346574685169114\n",
      "BACC total:  0.39250755409269955\n",
      "MCC total:  0.37738140398415526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    name = 'dnn'\n",
    "    pred_label = preds_dnn_01\n",
    "        \n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_dnn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "except: None    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_svm = time.time()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "clf.fit(X_train_01, y_train_01)\n",
    "end = time.time()\n",
    "clf.score(X_train_01, y_train_01)\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "joblib.dump(clf, 'svm_level_01.joblib')\n",
    "\n",
    "\n",
    "clf = loaded_model = joblib.load('svm_level_01.joblib')\n",
    "\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6537.0   162.0   169.0   12.0  0.0\n",
      "1.0  1302.0  3311.0   173.0   49.0  0.0\n",
      "2.0    43.0    10.0  1147.0   87.0  6.0\n",
      "3.0    11.0     0.0   114.0  224.0  0.0\n",
      "4.0     4.0     0.0     1.0    5.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8393057529737413\n",
      "Precision total:  0.617530344854312\n",
      "Recall total:  0.6327723610401879\n",
      "F1 total:  0.6179583394560351\n",
      "BACC total:  0.6327723610401879\n",
      "MCC total:  0.7378836756316417\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pred_label = preds_svm_01\n",
    "name = 'svm'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_svm\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6707.0    93.0    63.0   17.0  0.0\n",
      "1.0   224.0  4553.0    34.0   24.0  0.0\n",
      "2.0     6.0     5.0  1231.0   51.0  0.0\n",
      "3.0     0.0     7.0    12.0  330.0  0.0\n",
      "4.0     0.0     0.0     0.0   10.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9591531383257276\n",
      "Precision total:  0.7253696398208536\n",
      "Recall total:  0.7628276344191234\n",
      "F1 total:  0.7420383287420287\n",
      "BACC total:  0.7628276344191234\n",
      "MCC total:  0.9316962260398585\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_rf = time.time()\n",
    "\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "if True == True:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf_01 = rf.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(model_rf_01, X_train_01, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf_01, 'rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    model_rf_01  = joblib.load('rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)\n",
    "pred_label = preds_rf_01\n",
    "name='rf'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_rf\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0   4.0\n",
      "0.0  6791.0    57.0    26.0    6.0   0.0\n",
      "1.0   145.0  4652.0    17.0    7.0  14.0\n",
      "2.0    14.0    41.0  1227.0   11.0   0.0\n",
      "3.0     2.0    16.0    11.0  313.0   7.0\n",
      "4.0     0.0     0.0     1.0    2.0   7.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9717962145582404\n",
      "Precision total:  0.8166647772453425\n",
      "Recall total:  0.8990037979834724\n",
      "F1 total:  0.8364612346776681\n",
      "BACC total:  0.8990037979834724\n",
      "MCC total:  0.9525190955823462\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "start_lgbm = time.time()\n",
    "\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    lgbm = joblib.load('lgbm_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    pred_label = preds_lgbm_01\n",
    "    name='lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1       2      3    4\n",
      "0  6682.0   144.0    48.0    6.0  0.0\n",
      "1   448.0  4348.0    37.0    2.0  0.0\n",
      "2    19.0    12.0  1190.0   72.0  0.0\n",
      "3     5.0     4.0    16.0  322.0  2.0\n",
      "4     0.0     3.0     0.0    3.0  4.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9385800852846562\n",
      "Precision total:  0.8562766994376341\n",
      "Recall total:  0.8226946877920449\n",
      "F1 total:  0.8315748735075669\n",
      "BACC total:  0.8226946877920449\n",
      "MCC total:  0.8970919635408797\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_mlp = time.time()\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    MLP = joblib.load('mlp_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#MLP\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('MLP 01 model', file = f)\n",
    "    pred_label = preds_mlp_01\n",
    "    name='mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_mlp\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0    1.0     2.0   3.0  4.0\n",
      "0.0  6658.0  151.0    69.0   2.0  0.0\n",
      "1.0  4497.0  190.0   148.0   0.0  0.0\n",
      "2.0    40.0    2.0  1245.0   6.0  0.0\n",
      "3.0     3.0    3.0   258.0  85.0  0.0\n",
      "4.0     0.0    0.0     1.0   0.0  9.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.6124784918081844\n",
      "Precision total:  0.7562197039712557\n",
      "Recall total:  0.622691878221366\n",
      "F1 total:  0.5936118567118461\n",
      "BACC total:  0.622691878221366\n",
      "MCC total:  0.3899046349005824\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "# from sklearn.multioutput import MultiOutputClassifier\n",
    "start_ada = time.time()\n",
    "\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_01.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    ada = joblib.load('ada_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada_01 = ada.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('ADA 01 model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_ada_01\n",
    "    name='ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_ada\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6785.0    57.0    37.0    1.0  0.0\n",
      "1.0    51.0  4770.0    12.0    2.0  0.0\n",
      "2.0     8.0    10.0  1260.0   14.0  1.0\n",
      "3.0     6.0     2.0    11.0  330.0  0.0\n",
      "4.0     0.0     0.0     0.0    4.0  6.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9838408019750131\n",
      "Precision total:  0.9456222112626056\n",
      "Recall total:  0.8985569835665389\n",
      "F1 total:  0.917528669321017\n",
      "BACC total:  0.8985569835665389\n",
      "MCC total:  0.9728528424229709\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_knn = time.time()\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf_01, 'knn_01.joblib')\n",
    "\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf_01 = joblib.load('knn_01.joblib')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('KNN 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_knn\n",
    "    name='knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start_knn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6525.0   212.0   130.0   13.0  0.0\n",
      "1.0   882.0  3775.0    98.0   80.0  0.0\n",
      "2.0    30.0    19.0  1186.0   58.0  0.0\n",
      "3.0    10.0     0.0    92.0  247.0  0.0\n",
      "4.0     4.0     0.0     2.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8777586593850527\n",
      "Precision total:  0.643791576835065\n",
      "Recall total:  0.6708299037813601\n",
      "F1 total:  0.6538438356483864\n",
      "BACC total:  0.6708299037813601\n",
      "MCC total:  0.797878425654422\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "start_lr = time.time()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    logreg_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_logreg =logreg_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('LR 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_logreg\n",
    "    # pred_label = label[ypred]\n",
    "    name='lr'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.2945419\ttest: 1.2948974\tbest: 1.2948974 (0)\ttotal: 10.1ms\tremaining: 997ms\n",
      "10:\tlearn: 0.4187806\ttest: 0.4211433\tbest: 0.4211433 (10)\ttotal: 96.6ms\tremaining: 782ms\n",
      "20:\tlearn: 0.2135529\ttest: 0.2168026\tbest: 0.2168026 (20)\ttotal: 165ms\tremaining: 620ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30:\tlearn: 0.1356339\ttest: 0.1393834\tbest: 0.1393834 (30)\ttotal: 321ms\tremaining: 715ms\n",
      "40:\tlearn: 0.1021844\ttest: 0.1058540\tbest: 0.1058540 (40)\ttotal: 363ms\tremaining: 522ms\n",
      "50:\tlearn: 0.0832402\ttest: 0.0870315\tbest: 0.0870315 (50)\ttotal: 405ms\tremaining: 389ms\n",
      "60:\tlearn: 0.0706391\ttest: 0.0743694\tbest: 0.0743694 (60)\ttotal: 446ms\tremaining: 285ms\n",
      "70:\tlearn: 0.0635998\ttest: 0.0676387\tbest: 0.0676387 (70)\ttotal: 488ms\tremaining: 199ms\n",
      "80:\tlearn: 0.0573301\ttest: 0.0613835\tbest: 0.0613835 (80)\ttotal: 556ms\tremaining: 130ms\n",
      "90:\tlearn: 0.0532248\ttest: 0.0574554\tbest: 0.0574554 (90)\ttotal: 599ms\tremaining: 59.3ms\n",
      "99:\tlearn: 0.0501664\ttest: 0.0545865\tbest: 0.0545865 (99)\ttotal: 639ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.05458654743\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6818.0    34.0    23.0    5.0  0.0\n",
      "1.0    58.0  4760.0    14.0    3.0  0.0\n",
      "2.0     5.0     9.0  1256.0   21.0  2.0\n",
      "3.0     3.0     0.0     7.0  339.0  0.0\n",
      "4.0     0.0     0.0     1.0    2.0  7.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9860103239320716\n",
      "Precision total:  0.9281730058202363\n",
      "Recall total:  0.9236415123860129\n",
      "F1 total:  0.9253328448355616\n",
      "BACC total:  0.9236415123860129\n",
      "MCC total:  0.9764885225669266\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_01.fit(X_train_01, y_train_01, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('catboost', file = f)\n",
    "\n",
    "\n",
    "pred_label = preds_cat\n",
    "name='cat'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0     2.0    3.0  4.0\n",
      "0.0  6817.0    38.0    21.0    4.0  0.0\n",
      "1.0    71.0  4737.0    21.0    6.0  0.0\n",
      "2.0    12.0     5.0  1259.0   17.0  0.0\n",
      "3.0     3.0     1.0     5.0  339.0  1.0\n",
      "4.0     0.0     0.0     0.0    1.0  9.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9845889129946884\n",
      "Precision total:  0.953211305238258\n",
      "Recall total:  0.9631250836712203\n",
      "F1 total:  0.958036813549769\n",
      "BACC total:  0.9631250836712203\n",
      "MCC total:  0.9741023203300616\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import xgboost as xgb\n",
    "start = time.time()\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "    pred_label = preds_xgb_01\n",
    "    name='xgb'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Generating Summary Metric Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Models      |    ACC-01 |    PRE-01 |   REC-01 |    F1-01 |\n",
      "+=============+===========+===========+==========+==========+\n",
      "| XGB         | 0.984589  | 0.953211  | 0.963125 | 0.958037 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_DT      | 0.990948  | 0.944728  | 0.94406  | 0.944353 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_lgbm    | 0.990499  | 0.922579  | 0.928427 | 0.925473 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| CAT         | 0.98601   | 0.928173  | 0.923642 | 0.925333 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| KNN         | 0.983841  | 0.945622  | 0.898557 | 0.917529 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_knn     | 0.985262  | 0.925028  | 0.903572 | 0.912567 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_cat     | 0.985262  | 0.923071  | 0.903318 | 0.911391 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_mlp     | 0.931024  | 0.839735  | 0.846056 | 0.839822 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| LGBM        | 0.971796  | 0.816665  | 0.899004 | 0.836461 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| MLP         | 0.93858   | 0.856277  | 0.822695 | 0.831575 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_comb    | 0.947109  | 0.937143  | 0.774627 | 0.780994 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_rf      | 0.966634  | 0.72708   | 0.77005  | 0.745899 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| RF          | 0.959153  | 0.72537   | 0.762828 | 0.742038 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| LR          | 0.877759  | 0.643792  | 0.67083  | 0.653844 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_LR      | 0.876636  | 0.641174  | 0.667204 | 0.650694 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_svm     | 0.84155   | 0.613701  | 0.636088 | 0.618082 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| SVM         | 0.839306  | 0.61753   | 0.632772 | 0.617958 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| ADA         | 0.612478  | 0.75622   | 0.622692 | 0.593612 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| DNN         | 0.598938  | 0.24753   | 0.392508 | 0.303466 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| Bag_ada     | 0.0280542 | 0.455323  | 0.319593 | 0.206604 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| avg         | 0.358345  | 0.0947859 | 0.19745  | 0.110192 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| weighed_avg | 0.362909  | 0.131493  | 0.200354 | 0.107988 |\n",
      "+-------------+-----------+-----------+----------+----------+\n",
      "| VOTING      | 0         | 0         | 0        | 0        |\n",
      "+-------------+-----------+-----------+----------+----------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(5)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_acc = [\n",
    "                ada_acc_01,\n",
    "                svm_acc_01,\n",
    "                dnn_acc_01,\n",
    "                mlp_acc_01,\n",
    "                knn_acc_01,\n",
    "                cat_acc_01,\n",
    "                xgb_acc_01,\n",
    "                lgbm_acc_01,\n",
    "                rf_acc_01,\n",
    "                lr_acc_01,\n",
    "                voting_acc_01,\n",
    "                bag_svm_acc_01,\n",
    "                bag_knn_acc_01,\n",
    "                bag_dt_acc_01,\n",
    "                bag_lr_acc_01,\n",
    "                bag_mlp_acc_01,\n",
    "\n",
    "                bag_rf_acc_01,\n",
    "                bag_ada_acc_01,\n",
    "                bag_lgbm_acc_01,\n",
    "                # bag_xgb_acc_01,\n",
    "                bag_cat_acc_01,\n",
    "                bag_comb_acc_01,\n",
    "\n",
    "                avg_acc_01,\n",
    "                weighed_avg_acc_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "level_01_pre = [\n",
    "                ada_pre_01,\n",
    "                svm_pre_01,\n",
    "                dnn_pre_01,\n",
    "                mlp_pre_01,\n",
    "                knn_pre_01,\n",
    "                cat_pre_01,\n",
    "                xgb_pre_01,\n",
    "                lgbm_pre_01,\n",
    "                rf_pre_01,\n",
    "                lr_pre_01,\n",
    "                voting_pre_01,\n",
    "                bag_svm_pre_01,\n",
    "                bag_knn_pre_01,\n",
    "                bag_dt_pre_01,\n",
    "                bag_lr_pre_01,\n",
    "                bag_mlp_pre_01,\n",
    "\n",
    "                bag_rf_pre_01,\n",
    "                bag_ada_pre_01,\n",
    "                bag_lgbm_pre_01,\n",
    "                # bag_xgb_pre_01,\n",
    "                bag_cat_pre_01,\n",
    "                bag_comb_pre_01,\n",
    "\n",
    "                avg_pre_01,\n",
    "                weighed_avg_pre_01\n",
    "                ]  \n",
    "\n",
    "level_01_rec = [\n",
    "                ada_rec_01,\n",
    "                svm_rec_01,\n",
    "                dnn_rec_01,\n",
    "                mlp_rec_01,\n",
    "                knn_rec_01,\n",
    "                cat_rec_01,\n",
    "                xgb_rec_01,\n",
    "                lgbm_rec_01,\n",
    "                rf_rec_01,\n",
    "                lr_rec_01,\n",
    "                voting_rec_01,\n",
    "                bag_svm_rec_01,\n",
    "                bag_knn_rec_01,\n",
    "                bag_dt_rec_01,\n",
    "                bag_lr_rec_01,\n",
    "                bag_mlp_rec_01,\n",
    "\n",
    "                bag_rf_rec_01,\n",
    "                bag_ada_rec_01,\n",
    "                bag_lgbm_rec_01,\n",
    "                # bag_xgb_rec_01,\n",
    "                bag_cat_rec_01,\n",
    "                bag_comb_rec_01,\n",
    "\n",
    "                avg_rec_01,\n",
    "                weighed_avg_rec_01\n",
    "                ]  \n",
    "\n",
    "level_01_f1 = [\n",
    "                ada_f1_01,\n",
    "                svm_f1_01,\n",
    "                dnn_f1_01,\n",
    "                mlp_f1_01,\n",
    "                knn_f1_01,\n",
    "                cat_f1_01,\n",
    "                xgb_f1_01,\n",
    "                lgbm_f1_01,\n",
    "                rf_f1_01,\n",
    "                lr_f1_01,\n",
    "                voting_f1_01,\n",
    "                bag_svm_f1_01,\n",
    "                bag_knn_f1_01,\n",
    "                bag_dt_f1_01,\n",
    "                bag_lr_f1_01,\n",
    "                bag_mlp_f1_01,\n",
    "\n",
    "                bag_rf_f1_01,\n",
    "                bag_ada_f1_01,\n",
    "                bag_lgbm_f1_01,\n",
    "                # bag_xgb_f1_01,\n",
    "                bag_cat_f1_01,\n",
    "                bag_comb_f1_01,\n",
    "\n",
    "                avg_f1_01,\n",
    "                weighed_avg_f1_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_acc, level_01_pre, level_01_rec, level_01_f1))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_acc, sorted_level_01_pre, sorted_level_01_rec, sorted_level_01_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_acc[i]\n",
    "    data[i][2] = sorted_level_01_pre[i] \n",
    "    data[i][3] = sorted_level_01_rec[i] \n",
    "    data[i][4] = sorted_level_01_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-01\", \"PRE-01\", \"REC-01\", \"F1-01\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table', file = f)\n",
    "if pick_prob == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - Probabilities', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - CLASSES', file = f)\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------------+\n",
      "| Models      |   time-01(sec) |\n",
      "+=============+================+\n",
      "| avg         |       0.09532  |\n",
      "+-------------+----------------+\n",
      "| weighed_avg |       0.105209 |\n",
      "+-------------+----------------+\n",
      "| SVM         |       0.211002 |\n",
      "+-------------+----------------+\n",
      "| RF          |       0.32106  |\n",
      "+-------------+----------------+\n",
      "| KNN         |       0.429229 |\n",
      "+-------------+----------------+\n",
      "| Bag_DT      |       0.556273 |\n",
      "+-------------+----------------+\n",
      "| CAT         |       0.795552 |\n",
      "+-------------+----------------+\n",
      "| ADA         |       0.917045 |\n",
      "+-------------+----------------+\n",
      "| LR          |       1.22534  |\n",
      "+-------------+----------------+\n",
      "| Bag_knn     |       1.61397  |\n",
      "+-------------+----------------+\n",
      "| Bag_svm     |       2.01435  |\n",
      "+-------------+----------------+\n",
      "| Bag_rf      |       3.18566  |\n",
      "+-------------+----------------+\n",
      "| Bag_ada     |       6.37563  |\n",
      "+-------------+----------------+\n",
      "| Bag_cat     |       6.45565  |\n",
      "+-------------+----------------+\n",
      "| DNN         |       9.05407  |\n",
      "+-------------+----------------+\n",
      "| Bag_LR      |      10.3264   |\n",
      "+-------------+----------------+\n",
      "| MLP         |      33.2254   |\n",
      "+-------------+----------------+\n",
      "| XGB         |      62.6121   |\n",
      "+-------------+----------------+\n",
      "| Bag_mlp     |     416.981    |\n",
      "+-------------+----------------+\n",
      "| LGBM        |     462.938    |\n",
      "+-------------+----------------+\n",
      "| Bag_comb    |     529.088    |\n",
      "+-------------+----------------+\n",
      "| Bag_lgbm    |     741.341    |\n",
      "+-------------+----------------+\n",
      "| VOTING      |    9999        |\n",
      "+-------------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "from tabulate import tabulate\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_time = [\n",
    "                ada_time_01,\n",
    "                svm_time_01,\n",
    "                dnn_time_01,\n",
    "                mlp_time_01,\n",
    "                knn_time_01,\n",
    "                cat_time_01,\n",
    "                xgb_time_01,\n",
    "                lgbm_time_01,\n",
    "                rf_time_01,\n",
    "                lr_time_01,\n",
    "                voting_time_01,\n",
    "                bag_svm_time_01,\n",
    "                bag_knn_time_01,\n",
    "                bag_dt_time_01,\n",
    "                bag_lr_time_01,\n",
    "                bag_mlp_time_01,\n",
    "\n",
    "                bag_rf_time_01,\n",
    "                bag_ada_time_01,\n",
    "                bag_lgbm_time_01,\n",
    "                # bag_xgb_time_01,\n",
    "                bag_cat_time_01,\n",
    "                bag_comb_time_01,\n",
    "\n",
    "                avg_time_01,\n",
    "                weighed_avg_time_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_time))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-01(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  with open(output_file_name, \"a\") as f:print('ADA FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  # models = [ada,dnn_01,clf,knn_clf_01,cat_01,xgb_01, rf, lgbm, mlp,logreg_01]\n",
    "  explainer = shap.KernelExplainer(ada.predict_proba, test[start_index:end_index])\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                    # class_names=[column_features[:-1]],\n",
    "                    show=False)\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('ADA_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('ADA_SHAP_NSL_class_01.png')\n",
    "        \n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# explainer = shap.TreeExplainer(model)\n",
    "# start_index = 0\n",
    "# end_index = samples\n",
    "# shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "# shap_obj = explainer(test[start_index:end_index])\n",
    "# shap.summary_plot(shap_values = shap_values,\n",
    "#                   features = test[start_index:end_index],\n",
    "#                 show=False)\n",
    "# plt.savefig('Light_SHAP_CIC_Summary.png')\n",
    "# plt.clf()\n",
    "\n",
    "\n",
    "# vals= np.abs(shap_values).mean(1)\n",
    "# feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "# feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "# feature_importance.head()\n",
    "# print(feature_importance.to_string())\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "# feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "# feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# # for item1, item2 in zip(feature_name, feature_val):\n",
    "# #     print(item1, item2)\n",
    "\n",
    "\n",
    "# # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "# zipped_lists = list(zip(feature_name, feature_val))\n",
    "# zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# # Convert the sorted result back into separate lists\n",
    "# sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "# print('---------------------------------------------------------------------------------')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('XGB FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(xgb_01)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('XGB_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('XGB_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('LGBM FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(lgbm)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('LGBM_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('LGBM_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('RF FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(rf)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('RF_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('RF_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
